1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
9 // GC automatically manages memory allocated by managed code.
10 // The design doc for GC can be found at Documentation/botr/garbage-collection.md
12 // This file includes both the code for GC and the allocator. The most common
13 // case for a GC to be triggered is from the allocator code. See
14 // code:#try_allocate_more_space where it calls GarbageCollectGeneration.
16 // Entry points for the allocator are GCHeap::Alloc* which are called by the
17 // allocation helpers in gcscan.cpp
21 #include "softwarewritewatch.h"
25 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
26 BOOL bgc_heap_walk_for_etw_p = FALSE;
27 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
29 #if defined(FEATURE_REDHAWK)
30 #define MAYBE_UNUSED_VAR(v) v = v
32 #define MAYBE_UNUSED_VAR(v)
33 #endif // FEATURE_REDHAWK
35 #define MAX_PTR ((uint8_t*)(~(ptrdiff_t)0))
38 #define partial_size_th 100
39 #define num_partial_refs 64
41 #define partial_size_th 100
42 #define num_partial_refs 32
45 #define demotion_plug_len_th (6*1024*1024)
48 #define MARK_STACK_INITIAL_LENGTH 1024
50 #define MARK_STACK_INITIAL_LENGTH 128
53 #define LOH_PIN_QUEUE_LENGTH 100
54 #define LOH_PIN_DECAY 10
57 // Right now we support maximum 1024 procs - meaning that we will create at most
58 // that many GC threads and GC heaps.
59 #define MAX_SUPPORTED_CPUS 1024
61 #define MAX_SUPPORTED_CPUS 64
64 #ifdef GC_CONFIG_DRIVEN
65 int compact_ratio = 0;
66 #endif //GC_CONFIG_DRIVEN
68 // See comments in reset_memory.
69 BOOL reset_mm_p = TRUE;
71 bool g_fFinalizerRunOnShutDown = false;
74 bool g_built_with_svr_gc = true;
76 bool g_built_with_svr_gc = false;
77 #endif // FEATURE_SVR_GC
79 #if defined(BUILDENV_DEBUG)
80 uint8_t g_build_variant = 0;
81 #elif defined(BUILDENV_CHECKED)
82 uint8_t g_build_variant = 1;
84 uint8_t g_build_variant = 2;
85 #endif // defined(BUILDENV_DEBUG)
87 VOLATILE(int32_t) g_no_gc_lock = -1;
89 #if defined (TRACE_GC) && !defined (DACCESS_COMPILE)
90 const char * const allocation_state_str[] = {
96 "try_fit_new_seg_after_cg",
100 "try_free_full_seg_in_bgc",
101 "try_free_after_bgc",
104 "acquire_seg_after_cg",
105 "acquire_seg_after_bgc",
106 "check_and_wait_for_bgc",
107 "trigger_full_compact_gc",
108 "trigger_ephemeral_gc",
109 "trigger_2nd_ephemeral_gc",
112 #endif //TRACE_GC && !DACCESS_COMPILE
114 // Keep this in sync with the definition of gc_reason
115 #if (defined(DT_LOG) || defined(TRACE_GC)) && !defined (DACCESS_COMPILE)
116 static const char* const str_gc_reasons[] =
131 static const char* const str_gc_pause_modes[] =
136 "sustained_low_latency",
139 #endif // defined(DT_LOG) || defined(TRACE_GC)
142 BOOL is_induced (gc_reason reason)
144 return ((reason == reason_induced) ||
145 (reason == reason_induced_noforce) ||
146 (reason == reason_lowmemory) ||
147 (reason == reason_lowmemory_blocking) ||
148 (reason == reason_induced_compacting));
152 BOOL is_induced_blocking (gc_reason reason)
154 return ((reason == reason_induced) ||
155 (reason == reason_lowmemory_blocking) ||
156 (reason == reason_induced_compacting));
159 #ifndef DACCESS_COMPILE
162 size_t GetHighPrecisionTimeStamp()
164 int64_t ts = GCToOSInterface::QueryPerformanceCounter();
166 return (size_t)(ts / (qpf / 1000));
172 // There is a current and a prior copy of the statistics. This allows us to display deltas per reporting
173 // interval, as well as running totals. The 'min' and 'max' values require special treatment. They are
174 // Reset (zeroed) in the current statistics when we begin a new interval and they are updated via a
175 // comparison with the global min/max.
176 GCStatistics g_GCStatistics;
177 GCStatistics g_LastGCStatistics;
179 char* GCStatistics::logFileName = NULL;
180 FILE* GCStatistics::logFile = NULL;
182 void GCStatistics::AddGCStats(const gc_mechanisms& settings, size_t timeInMSec)
185 if (settings.concurrent)
187 bgc.Accumulate((uint32_t)timeInMSec*1000);
190 else if (settings.background_p)
192 fgc.Accumulate((uint32_t)timeInMSec*1000);
194 if (settings.compaction)
196 assert(settings.condemned_generation < max_generation);
197 cntFGCGen[settings.condemned_generation]++;
200 #endif // BACKGROUND_GC
202 ngc.Accumulate((uint32_t)timeInMSec*1000);
204 if (settings.compaction)
206 cntNGCGen[settings.condemned_generation]++;
209 if (is_induced (settings.reason))
210 cntReasons[(int)reason_induced]++;
212 else if (settings.stress_induced)
213 cntReasons[(int)reason_gcstress]++;
214 #endif // STRESS_HEAP
216 cntReasons[(int)settings.reason]++;
219 if (settings.concurrent || !settings.background_p)
221 #endif // BACKGROUND_GC
225 #endif // BACKGROUND_GC
228 void GCStatistics::Initialize()
230 LIMITED_METHOD_CONTRACT;
231 // for efficiency sake we're taking a dependency on the layout of a C++ object
232 // with a vtable. protect against violations of our premise:
233 static_assert(offsetof(GCStatistics, cntDisplay) == sizeof(void*),
234 "The first field of GCStatistics follows the pointer sized vtable");
236 int podOffs = offsetof(GCStatistics, cntDisplay); // offset of the first POD field
237 memset((uint8_t*)(&g_GCStatistics)+podOffs, 0, sizeof(g_GCStatistics)-podOffs);
238 memset((uint8_t*)(&g_LastGCStatistics)+podOffs, 0, sizeof(g_LastGCStatistics)-podOffs);
241 void GCStatistics::DisplayAndUpdate()
243 LIMITED_METHOD_CONTRACT;
245 if (logFileName == NULL || logFile == NULL)
250 fprintf(logFile, "\nGCMix **** Initialize *****\n\n");
252 fprintf(logFile, "GCMix **** Summary ***** %d\n", cntDisplay);
254 // NGC summary (total, timing info)
255 ngc.DisplayAndUpdate(logFile, "NGC ", &g_LastGCStatistics.ngc, cntNGC, g_LastGCStatistics.cntNGC, msec);
257 // FGC summary (total, timing info)
258 fgc.DisplayAndUpdate(logFile, "FGC ", &g_LastGCStatistics.fgc, cntFGC, g_LastGCStatistics.cntFGC, msec);
261 bgc.DisplayAndUpdate(logFile, "BGC ", &g_LastGCStatistics.bgc, cntBGC, g_LastGCStatistics.cntBGC, msec);
263 // NGC/FGC break out by generation & compacting vs. sweeping
264 fprintf(logFile, "NGC ");
265 for (int i = max_generation; i >= 0; --i)
266 fprintf(logFile, "gen%d %d (%d). ", i, cntNGCGen[i]-g_LastGCStatistics.cntNGCGen[i], cntNGCGen[i]);
267 fprintf(logFile, "\n");
269 fprintf(logFile, "FGC ");
270 for (int i = max_generation-1; i >= 0; --i)
271 fprintf(logFile, "gen%d %d (%d). ", i, cntFGCGen[i]-g_LastGCStatistics.cntFGCGen[i], cntFGCGen[i]);
272 fprintf(logFile, "\n");
274 // Compacting vs. Sweeping break out
275 int _cntSweep = cntNGC-cntCompactNGC;
276 int _cntLastSweep = g_LastGCStatistics.cntNGC-g_LastGCStatistics.cntCompactNGC;
277 fprintf(logFile, "NGC Sweeping %d (%d) Compacting %d (%d)\n",
278 _cntSweep - _cntLastSweep, _cntSweep,
279 cntCompactNGC - g_LastGCStatistics.cntCompactNGC, cntCompactNGC);
281 _cntSweep = cntFGC-cntCompactFGC;
282 _cntLastSweep = g_LastGCStatistics.cntFGC-g_LastGCStatistics.cntCompactFGC;
283 fprintf(logFile, "FGC Sweeping %d (%d) Compacting %d (%d)\n",
284 _cntSweep - _cntLastSweep, _cntSweep,
285 cntCompactFGC - g_LastGCStatistics.cntCompactFGC, cntCompactFGC);
289 for (int reason=(int)reason_alloc_soh; reason <= (int)reason_gcstress; ++reason)
291 if (cntReasons[reason] != 0)
292 fprintf(logFile, "%s %d (%d). ", str_gc_reasons[reason],
293 cntReasons[reason]-g_LastGCStatistics.cntReasons[reason], cntReasons[reason]);
296 fprintf(logFile, "\n\n");
298 // flush the log file...
302 g_LastGCStatistics = *this;
312 #define TOTAL_TIMES_TO_SHIFT 6
314 #define TOTAL_TIMES_TO_SHIFT 5
317 size_t round_up_power2 (size_t size)
319 unsigned short shift = 1;
323 for (unsigned short i = 0; i < TOTAL_TIMES_TO_SHIFT; i++)
325 shifted = size | (size >> shift);
340 size_t round_down_power2 (size_t size)
342 size_t power2 = round_up_power2 (size);
352 // the index starts from 0.
353 int index_of_set_bit (size_t power2)
356 int high = sizeof (size_t) * 8 - 1;
360 mid = ((low + high)/2);
361 size_t temp = (size_t)1 << mid;
366 else if (power2 < temp)
380 int relative_index_power2_plug (size_t power2)
382 int index = index_of_set_bit (power2);
383 assert (index <= MAX_INDEX_POWER2);
385 return ((index < MIN_INDEX_POWER2) ? 0 : (index - MIN_INDEX_POWER2));
389 int relative_index_power2_free_space (size_t power2)
391 int index = index_of_set_bit (power2);
392 assert (index <= MAX_INDEX_POWER2);
394 return ((index < MIN_INDEX_POWER2) ? -1 : (index - MIN_INDEX_POWER2));
398 uint32_t bgc_alloc_spin_count = 140;
399 uint32_t bgc_alloc_spin_count_loh = 16;
400 uint32_t bgc_alloc_spin = 2;
404 void c_write (uint32_t& place, uint32_t value)
406 Interlocked::Exchange (&place, value);
410 #ifndef DACCESS_COMPILE
411 // If every heap's gen2 or gen3 size is less than this threshold we will do a blocking GC.
412 const size_t bgc_min_per_heap = 4*1024*1024;
414 int gc_heap::gchist_index = 0;
415 gc_mechanisms_store gc_heap::gchist[max_history_count];
417 #ifndef MULTIPLE_HEAPS
418 size_t gc_heap::total_promoted_bytes = 0;
419 VOLATILE(bgc_state) gc_heap::current_bgc_state = bgc_not_in_process;
420 int gc_heap::gchist_index_per_heap = 0;
421 gc_heap::gc_history gc_heap::gchist_per_heap[max_history_count];
422 #endif //MULTIPLE_HEAPS
424 void gc_heap::add_to_history_per_heap()
427 gc_history* current_hist = &gchist_per_heap[gchist_index_per_heap];
428 current_hist->gc_index = settings.gc_index;
429 current_hist->current_bgc_state = current_bgc_state;
430 size_t elapsed = dd_gc_elapsed_time (dynamic_data_of (0));
431 current_hist->gc_time_ms = (uint32_t)elapsed;
432 current_hist->gc_efficiency = (elapsed ? (total_promoted_bytes / elapsed) : total_promoted_bytes);
433 current_hist->eph_low = generation_allocation_start (generation_of (max_generation-1));
434 current_hist->gen0_start = generation_allocation_start (generation_of (0));
435 current_hist->eph_high = heap_segment_allocated (ephemeral_heap_segment);
437 current_hist->bgc_lowest = background_saved_lowest_address;
438 current_hist->bgc_highest = background_saved_highest_address;
439 #endif //BACKGROUND_GC
440 current_hist->fgc_lowest = lowest_address;
441 current_hist->fgc_highest = highest_address;
442 current_hist->g_lowest = g_gc_lowest_address;
443 current_hist->g_highest = g_gc_highest_address;
445 gchist_index_per_heap++;
446 if (gchist_index_per_heap == max_history_count)
448 gchist_index_per_heap = 0;
453 void gc_heap::add_to_history()
456 gc_mechanisms_store* current_settings = &gchist[gchist_index];
457 current_settings->store (&settings);
460 if (gchist_index == max_history_count)
467 #endif //DACCESS_COMPILE
468 #endif //BACKGROUND_GC
470 #if defined(TRACE_GC) && !defined(DACCESS_COMPILE)
471 BOOL gc_log_on = TRUE;
473 size_t gc_log_file_size = 0;
475 size_t gc_buffer_index = 0;
476 size_t max_gc_buffers = 0;
478 static CLRCriticalSection gc_log_lock;
480 // we keep this much in a buffer and only flush when the buffer is full
481 #define gc_log_buffer_size (1024*1024)
482 uint8_t* gc_log_buffer = 0;
483 size_t gc_log_buffer_offset = 0;
485 void log_va_msg(const char *fmt, va_list args)
489 const int BUFFERSIZE = 512;
490 static char rgchBuffer[BUFFERSIZE];
491 char * pBuffer = &rgchBuffer[0];
494 int buffer_start = 1;
495 int pid_len = sprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, "[%5d]", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging());
496 buffer_start += pid_len;
497 memset(&pBuffer[buffer_start], '-', BUFFERSIZE - buffer_start);
498 int msg_len = _vsnprintf_s(&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args );
501 msg_len = BUFFERSIZE - buffer_start;
504 msg_len += buffer_start;
506 if ((gc_log_buffer_offset + msg_len) > (gc_log_buffer_size - 12))
509 memset (index_str, '-', 8);
510 sprintf_s (index_str, _countof(index_str), "%d", (int)gc_buffer_index);
511 gc_log_buffer[gc_log_buffer_offset] = '\n';
512 memcpy (gc_log_buffer + (gc_log_buffer_offset + 1), index_str, 8);
515 if (gc_buffer_index > max_gc_buffers)
517 fseek (gc_log, 0, SEEK_SET);
520 fwrite(gc_log_buffer, gc_log_buffer_size, 1, gc_log);
522 memset (gc_log_buffer, '*', gc_log_buffer_size);
523 gc_log_buffer_offset = 0;
526 memcpy (gc_log_buffer + gc_log_buffer_offset, pBuffer, msg_len);
527 gc_log_buffer_offset += msg_len;
532 void GCLog (const char *fmt, ... )
534 if (gc_log_on && (gc_log != NULL))
538 log_va_msg (fmt, args);
542 #endif // TRACE_GC && !DACCESS_COMPILE
544 #if defined(GC_CONFIG_DRIVEN) && !defined(DACCESS_COMPILE)
546 BOOL gc_config_log_on = FALSE;
547 FILE* gc_config_log = NULL;
549 // we keep this much in a buffer and only flush when the buffer is full
550 #define gc_config_log_buffer_size (1*1024) // TEMP
551 uint8_t* gc_config_log_buffer = 0;
552 size_t gc_config_log_buffer_offset = 0;
554 // For config since we log so little we keep the whole history. Also it's only
555 // ever logged by one thread so no need to synchronize.
556 void log_va_msg_config(const char *fmt, va_list args)
558 const int BUFFERSIZE = 256;
559 static char rgchBuffer[BUFFERSIZE];
560 char * pBuffer = &rgchBuffer[0];
563 int buffer_start = 1;
564 int msg_len = _vsnprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args );
565 assert (msg_len != -1);
566 msg_len += buffer_start;
568 if ((gc_config_log_buffer_offset + msg_len) > gc_config_log_buffer_size)
570 fwrite(gc_config_log_buffer, gc_config_log_buffer_offset, 1, gc_config_log);
571 fflush(gc_config_log);
572 gc_config_log_buffer_offset = 0;
575 memcpy (gc_config_log_buffer + gc_config_log_buffer_offset, pBuffer, msg_len);
576 gc_config_log_buffer_offset += msg_len;
579 void GCLogConfig (const char *fmt, ... )
581 if (gc_config_log_on && (gc_config_log != NULL))
584 va_start( args, fmt );
585 log_va_msg_config (fmt, args);
588 #endif // GC_CONFIG_DRIVEN && !DACCESS_COMPILE
590 #ifdef SYNCHRONIZATION_STATS
592 // Number of GCs have we done since we last logged.
593 static unsigned int gc_count_during_log;
594 // In ms. This is how often we print out stats.
595 static const unsigned int log_interval = 5000;
596 // Time (in ms) when we start a new log interval.
597 static unsigned int log_start_tick;
598 static unsigned int gc_lock_contended;
599 static int64_t log_start_hires;
600 // Cycles accumulated in SuspendEE during log_interval.
601 static uint64_t suspend_ee_during_log;
602 // Cycles accumulated in RestartEE during log_interval.
603 static uint64_t restart_ee_during_log;
604 static uint64_t gc_during_log;
606 #endif //SYNCHRONIZATION_STATS
609 init_sync_log_stats()
611 #ifdef SYNCHRONIZATION_STATS
612 if (gc_count_during_log == 0)
614 gc_heap::init_sync_stats();
615 suspend_ee_during_log = 0;
616 restart_ee_during_log = 0;
618 gc_lock_contended = 0;
620 log_start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
621 log_start_hires = GCToOSInterface::QueryPerformanceCounter();
623 gc_count_during_log++;
624 #endif //SYNCHRONIZATION_STATS
628 process_sync_log_stats()
630 #ifdef SYNCHRONIZATION_STATS
632 unsigned int log_elapsed = GCToOSInterface::GetLowPrecisionTimeStamp() - log_start_tick;
634 if (log_elapsed > log_interval)
636 uint64_t total = GCToOSInterface::QueryPerformanceCounter() - log_start_hires;
637 // Print out the cycles we spent on average in each suspend and restart.
638 printf("\n_________________________________________________________________________________\n"
639 "Past %d(s): #%3d GCs; Total gc_lock contended: %8u; GC: %12u\n"
640 "SuspendEE: %8u; RestartEE: %8u GC %.3f%%\n",
644 (unsigned int)(gc_during_log / gc_count_during_log),
645 (unsigned int)(suspend_ee_during_log / gc_count_during_log),
646 (unsigned int)(restart_ee_during_log / gc_count_during_log),
647 (double)(100.0f * gc_during_log / total));
648 gc_heap::print_sync_stats(gc_count_during_log);
650 gc_count_during_log = 0;
652 #endif //SYNCHRONIZATION_STATS
655 #ifdef MULTIPLE_HEAPS
659 gc_join_init_cpu_mapping = 0,
661 gc_join_generation_determined = 2,
662 gc_join_begin_mark_phase = 3,
663 gc_join_scan_dependent_handles = 4,
664 gc_join_rescan_dependent_handles = 5,
665 gc_join_scan_sizedref_done = 6,
666 gc_join_null_dead_short_weak = 7,
667 gc_join_scan_finalization = 8,
668 gc_join_null_dead_long_weak = 9,
669 gc_join_null_dead_syncblk = 10,
670 gc_join_decide_on_compaction = 11,
671 gc_join_rearrange_segs_compaction = 12,
672 gc_join_adjust_handle_age_compact = 13,
673 gc_join_adjust_handle_age_sweep = 14,
674 gc_join_begin_relocate_phase = 15,
675 gc_join_relocate_phase_done = 16,
676 gc_join_verify_objects_done = 17,
677 gc_join_start_bgc = 18,
678 gc_join_restart_ee = 19,
679 gc_join_concurrent_overflow = 20,
680 gc_join_suspend_ee = 21,
681 gc_join_bgc_after_ephemeral = 22,
682 gc_join_allow_fgc = 23,
683 gc_join_bgc_sweep = 24,
684 gc_join_suspend_ee_verify = 25,
685 gc_join_restart_ee_verify = 26,
686 gc_join_set_state_free = 27,
687 gc_r_join_update_card_bundle = 28,
688 gc_join_after_absorb = 29,
689 gc_join_verify_copy_table = 30,
690 gc_join_after_reset = 31,
691 gc_join_after_ephemeral_sweep = 32,
692 gc_join_after_profiler_heap_walk = 33,
693 gc_join_minimal_gc = 34,
694 gc_join_after_commit_soh_no_gc = 35,
695 gc_join_expand_loh_no_gc = 36,
696 gc_join_final_no_gc = 37,
697 gc_join_disable_software_write_watch = 38,
703 join_flavor_server_gc = 0,
707 #define first_thread_arrived 2
708 struct DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE) join_structure
710 // Shared non volatile keep on separate line to prevent eviction
713 // Keep polling/wait structures on separate line write once per join
714 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
715 GCEvent joined_event[3]; // the last event in the array is only used for first_thread_arrived.
716 Volatile<int> lock_color;
717 VOLATILE(BOOL) wait_done;
718 VOLATILE(BOOL) joined_p;
720 // Keep volatile counted locks on separate cache line write many per join
721 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
722 VOLATILE(int32_t) join_lock;
723 VOLATILE(int32_t) r_join_lock;
732 type_first_r_join = 3,
744 join_heap_restart = 100,
745 join_heap_r_restart = 200
757 join_structure join_struct;
760 gc_join_flavor flavor;
763 uint64_t start[MAX_SUPPORTED_CPUS], end[MAX_SUPPORTED_CPUS], start_seq;
764 // remember join id and last thread to arrive so restart can use these
766 // we want to print statistics every 10 seconds - this is to remember the start of the 10 sec interval
768 // counters for joins, in 1000's of clock cycles
769 uint64_t elapsed_total[gc_join_max], wake_total[gc_join_max], seq_loss_total[gc_join_max], par_loss_total[gc_join_max], in_join_total[gc_join_max];
773 BOOL init (int n_th, gc_join_flavor f)
775 dprintf (JOIN_LOG, ("Initializing join structure"));
776 join_struct.n_threads = n_th;
777 join_struct.lock_color = 0;
778 for (int i = 0; i < 3; i++)
780 if (!join_struct.joined_event[i].IsValid())
782 join_struct.joined_p = FALSE;
783 dprintf (JOIN_LOG, ("Creating join event %d", i));
784 // TODO - changing this to a non OS event
785 // because this is also used by BGC threads which are
786 // managed threads and WaitEx does not allow you to wait
787 // for an OS event on a managed thread.
788 // But we are not sure if this plays well in the hosting
790 //join_struct.joined_event[i].CreateOSManualEventNoThrow(FALSE);
791 if (!join_struct.joined_event[i].CreateManualEventNoThrow(FALSE))
795 join_struct.join_lock = join_struct.n_threads;
796 join_struct.r_join_lock = join_struct.n_threads;
797 join_struct.wait_done = FALSE;
801 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
809 dprintf (JOIN_LOG, ("Destroying join structure"));
810 for (int i = 0; i < 3; i++)
812 if (join_struct.joined_event[i].IsValid())
813 join_struct.joined_event[i].CloseEvent();
817 inline void fire_event (int heap, join_time time, join_type type, int join_id)
819 FIRE_EVENT(GCJoin_V2, heap, time, type, join_id);
822 void join (gc_heap* gch, int join_id)
825 // parallel execution ends here
826 end[gch->heap_number] = get_ts();
829 assert (!join_struct.joined_p);
830 int color = join_struct.lock_color.LoadWithoutBarrier();
832 if (Interlocked::Decrement(&join_struct.join_lock) != 0)
834 dprintf (JOIN_LOG, ("join%d(%d): Join() Waiting...join_lock is now %d",
835 flavor, join_id, (int32_t)(join_struct.join_lock)));
837 fire_event (gch->heap_number, time_start, type_join, join_id);
839 //busy wait around the color
840 if (color == join_struct.lock_color.LoadWithoutBarrier())
843 int spin_count = 4096 * (gc_heap::n_heaps - 1);
844 for (int j = 0; j < spin_count; j++)
846 if (color != join_struct.lock_color.LoadWithoutBarrier())
850 YieldProcessor(); // indicate to the processor that we are spinning
853 // we've spun, and if color still hasn't changed, fall into hard wait
854 if (color == join_struct.lock_color.LoadWithoutBarrier())
856 dprintf (JOIN_LOG, ("join%d(%d): Join() hard wait on reset event %d, join_lock is now %d",
857 flavor, join_id, color, (int32_t)(join_struct.join_lock)));
859 //Thread* current_thread = GCToEEInterface::GetThread();
860 //BOOL cooperative_mode = gc_heap::enable_preemptive (current_thread);
861 uint32_t dwJoinWait = join_struct.joined_event[color].Wait(INFINITE, FALSE);
862 //gc_heap::disable_preemptive (current_thread, cooperative_mode);
864 if (dwJoinWait != WAIT_OBJECT_0)
866 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
871 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
872 if (color == join_struct.lock_color.LoadWithoutBarrier())
877 dprintf (JOIN_LOG, ("join%d(%d): Join() done, join_lock is %d",
878 flavor, join_id, (int32_t)(join_struct.join_lock)));
881 fire_event (gch->heap_number, time_end, type_join, join_id);
884 // parallel execution starts here
885 start[gch->heap_number] = get_ts();
886 Interlocked::ExchangeAdd(&in_join_total[join_id], (start[gch->heap_number] - end[gch->heap_number]));
891 fire_event (gch->heap_number, time_start, type_last_join, join_id);
893 join_struct.joined_p = TRUE;
894 dprintf (JOIN_LOG, ("join%d(%d): Last thread to complete the join, setting id", flavor, join_id));
895 join_struct.joined_event[!color].Reset();
897 // this one is alone so it can proceed
899 // remember the join id, the last thread arriving, the start of the sequential phase,
900 // and keep track of the cycles spent waiting in the join
901 thd = gch->heap_number;
902 start_seq = get_ts();
903 Interlocked::ExchangeAdd(&in_join_total[join_id], (start_seq - end[gch->heap_number]));
908 // Reverse join - first thread gets here does the work; other threads will only proceed
909 // after the work is done.
910 // Note that you cannot call this twice in a row on the same thread. Plus there's no
911 // need to call it twice in row - you should just merge the work.
912 BOOL r_join (gc_heap* gch, int join_id)
915 if (join_struct.n_threads == 1)
920 if (Interlocked::CompareExchange(&join_struct.r_join_lock, 0, join_struct.n_threads) == 0)
922 if (!join_struct.wait_done)
924 dprintf (JOIN_LOG, ("r_join() Waiting..."));
926 fire_event (gch->heap_number, time_start, type_join, join_id);
928 //busy wait around the color
929 if (!join_struct.wait_done)
932 int spin_count = 2 * 4096 * (gc_heap::n_heaps - 1);
933 for (int j = 0; j < spin_count; j++)
935 if (join_struct.wait_done)
939 YieldProcessor(); // indicate to the processor that we are spinning
942 // we've spun, and if color still hasn't changed, fall into hard wait
943 if (!join_struct.wait_done)
945 dprintf (JOIN_LOG, ("Join() hard wait on reset event %d", first_thread_arrived));
946 uint32_t dwJoinWait = join_struct.joined_event[first_thread_arrived].Wait(INFINITE, FALSE);
947 if (dwJoinWait != WAIT_OBJECT_0)
949 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
954 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
955 if (!join_struct.wait_done)
960 dprintf (JOIN_LOG, ("r_join() done"));
963 fire_event (gch->heap_number, time_end, type_join, join_id);
970 fire_event (gch->heap_number, time_start, type_first_r_join, join_id);
978 return GCToOSInterface::QueryPerformanceCounter();
981 void start_ts (gc_heap* gch)
983 // parallel execution ends here
984 start[gch->heap_number] = get_ts();
991 uint64_t elapsed_seq = get_ts() - start_seq;
992 uint64_t max = 0, sum = 0, wake = 0;
993 uint64_t min_ts = start[0];
994 for (int i = 1; i < join_struct.n_threads; i++)
996 if(min_ts > start[i]) min_ts = start[i];
999 for (int i = 0; i < join_struct.n_threads; i++)
1001 uint64_t wake_delay = start[i] - min_ts;
1002 uint64_t elapsed = end[i] - start[i];
1008 uint64_t seq_loss = (join_struct.n_threads - 1)*elapsed_seq;
1009 uint64_t par_loss = join_struct.n_threads*max - sum;
1010 double efficiency = 0.0;
1012 efficiency = sum*100.0/(join_struct.n_threads*max);
1014 const double ts_scale = 1e-6;
1016 // enable this printf to get statistics on each individual join as it occurs
1017 // printf("join #%3d seq_loss = %5g par_loss = %5g efficiency = %3.0f%%\n", join_id, ts_scale*seq_loss, ts_scale*par_loss, efficiency);
1019 elapsed_total[id] += sum;
1020 wake_total[id] += wake;
1021 seq_loss_total[id] += seq_loss;
1022 par_loss_total[id] += par_loss;
1024 // every 10 seconds, print a summary of the time spent in each type of join
1025 if (GCToOSInterface::GetLowPrecisionTimeStamp() - start_tick > 10*1000)
1027 printf("**** summary *****\n");
1028 for (int i = 0; i < 16; i++)
1030 printf("join #%3d elapsed_total = %8g wake_loss = %8g seq_loss = %8g par_loss = %8g in_join_total = %8g\n",
1032 ts_scale*elapsed_total[i],
1033 ts_scale*wake_total[i],
1034 ts_scale*seq_loss_total[i],
1035 ts_scale*par_loss_total[i],
1036 ts_scale*in_join_total[i]);
1037 elapsed_total[i] = wake_total[i] = seq_loss_total[i] = par_loss_total[i] = in_join_total[i] = 0;
1039 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
1043 fire_event (join_heap_restart, time_start, type_restart, -1);
1044 assert (join_struct.joined_p);
1045 join_struct.joined_p = FALSE;
1046 join_struct.join_lock = join_struct.n_threads;
1047 dprintf (JOIN_LOG, ("join%d(%d): Restarting from join: join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1048 // printf("restart from join #%d at cycle %u from start of gc\n", join_id, GetCycleCount32() - gc_start);
1049 int color = join_struct.lock_color.LoadWithoutBarrier();
1050 join_struct.lock_color = !color;
1051 join_struct.joined_event[color].Set();
1053 // printf("Set joined_event %d\n", !join_struct.lock_color);
1055 fire_event (join_heap_restart, time_end, type_restart, -1);
1058 start[thd] = get_ts();
1064 dprintf (JOIN_LOG, ("join%d(%d): joined, join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1065 return join_struct.joined_p;
1070 if (join_struct.n_threads != 1)
1072 fire_event (join_heap_r_restart, time_start, type_restart, -1);
1073 join_struct.wait_done = TRUE;
1074 join_struct.joined_event[first_thread_arrived].Set();
1075 fire_event (join_heap_r_restart, time_end, type_restart, -1);
1081 if (join_struct.n_threads != 1)
1083 join_struct.r_join_lock = join_struct.n_threads;
1084 join_struct.wait_done = FALSE;
1085 join_struct.joined_event[first_thread_arrived].Reset();
1092 #ifdef BACKGROUND_GC
1094 #endif //BACKGROUND_GC
1096 #endif //MULTIPLE_HEAPS
1098 #define spin_and_switch(count_to_spin, expr) \
1100 for (int j = 0; j < count_to_spin; j++) \
1110 GCToOSInterface::YieldThread(0); \
1114 #ifndef DACCESS_COMPILE
1115 #ifdef BACKGROUND_GC
1117 #define max_pending_allocs 64
1119 class exclusive_sync
1121 // TODO - verify that this is the right syntax for Volatile.
1122 VOLATILE(uint8_t*) rwp_object;
1123 VOLATILE(int32_t) needs_checking;
1127 uint8_t cache_separator[HS_CACHE_LINE_SIZE - sizeof (int) - sizeof (int32_t)];
1129 // TODO - perhaps each object should be on its own cache line...
1130 VOLATILE(uint8_t*) alloc_objects[max_pending_allocs];
1132 int find_free_index ()
1134 for (int i = 0; i < max_pending_allocs; i++)
1136 if (alloc_objects [i] == (uint8_t*)0)
1148 spin_count = 32 * (g_num_processors - 1);
1151 for (int i = 0; i < max_pending_allocs; i++)
1153 alloc_objects [i] = (uint8_t*)0;
1159 for (int i = 0; i < max_pending_allocs; i++)
1161 if (alloc_objects [i] != (uint8_t*)0)
1163 GCToOSInterface::DebugBreak();
1168 void bgc_mark_set (uint8_t* obj)
1170 dprintf (3, ("cm: probing %Ix", obj));
1172 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1174 // If we spend too much time spending all the allocs,
1175 // consider adding a high water mark and scan up
1176 // to that; we'll need to interlock in done when
1177 // we update the high watermark.
1178 for (int i = 0; i < max_pending_allocs; i++)
1180 if (obj == alloc_objects[i])
1183 dprintf (3, ("cm: will spin"));
1184 spin_and_switch (spin_count, (obj != alloc_objects[i]));
1191 dprintf (3, ("cm: set %Ix", obj));
1196 spin_and_switch (spin_count, (needs_checking == 0));
1201 int loh_alloc_set (uint8_t* obj)
1203 if (!gc_heap::cm_in_progress)
1209 dprintf (3, ("loh alloc: probing %Ix", obj));
1211 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1213 if (obj == rwp_object)
1216 spin_and_switch (spin_count, (obj != rwp_object));
1221 int cookie = find_free_index();
1225 alloc_objects[cookie] = obj;
1229 // GCToOSInterface::DebugBreak();
1232 dprintf (3, ("loh alloc: set %Ix at %d", obj, cookie));
1238 dprintf (3, ("loh alloc: setting %Ix will spin to acquire a free index", obj));
1239 spin_and_switch (spin_count, (find_free_index () != -1));
1246 dprintf (3, ("loh alloc: will spin on checking %Ix", obj));
1247 spin_and_switch (spin_count, (needs_checking == 0));
1252 void bgc_mark_done ()
1254 dprintf (3, ("cm: release lock on %Ix", (uint8_t *)rwp_object));
1258 void loh_alloc_done_with_index (int index)
1260 dprintf (3, ("loh alloc: release lock on %Ix based on %d", (uint8_t *)alloc_objects[index], index));
1261 assert ((index >= 0) && (index < max_pending_allocs));
1262 alloc_objects[index] = (uint8_t*)0;
1265 void loh_alloc_done (uint8_t* obj)
1267 #ifdef BACKGROUND_GC
1268 if (!gc_heap::cm_in_progress)
1273 for (int i = 0; i < max_pending_allocs; i++)
1275 if (alloc_objects [i] == obj)
1277 dprintf (3, ("loh alloc: release lock on %Ix at %d", (uint8_t *)alloc_objects[i], i));
1278 alloc_objects[i] = (uint8_t*)0;
1282 #endif //BACKGROUND_GC
1286 // Note that this class was written assuming just synchronization between
1287 // one background GC thread and multiple user threads that might request
1288 // an FGC - it does not take into account what kind of locks the multiple
1289 // user threads might be holding at the time (eg, there could only be one
1290 // user thread requesting an FGC because it needs to take gc_lock first)
1291 // so you'll see checks that may not be necessary if you take those conditions
1292 // into consideration.
1294 // With the introduction of Server Background GC we no longer use this
1295 // class to do synchronization between FGCs and BGC.
1296 class recursive_gc_sync
1298 static VOLATILE(int32_t) foreground_request_count;//initial state 0
1299 static VOLATILE(BOOL) gc_background_running; //initial state FALSE
1300 static VOLATILE(int32_t) foreground_count; // initial state 0;
1301 static VOLATILE(uint32_t) foreground_gate; // initial state FALSE;
1302 static GCEvent foreground_complete;//Auto Reset
1303 static GCEvent foreground_allowed;//Auto Reset
1305 static void begin_background();
1306 static void end_background();
1307 static void begin_foreground();
1308 static void end_foreground();
1309 BOOL allow_foreground ();
1311 static void shutdown();
1312 static BOOL background_running_p() {return gc_background_running;}
1315 VOLATILE(int32_t) recursive_gc_sync::foreground_request_count = 0;//initial state 0
1316 VOLATILE(int32_t) recursive_gc_sync::foreground_count = 0; // initial state 0;
1317 VOLATILE(BOOL) recursive_gc_sync::gc_background_running = FALSE; //initial state FALSE
1318 VOLATILE(uint32_t) recursive_gc_sync::foreground_gate = 0;
1319 GCEvent recursive_gc_sync::foreground_complete;//Auto Reset
1320 GCEvent recursive_gc_sync::foreground_allowed;//Manual Reset
1322 BOOL recursive_gc_sync::init ()
1324 foreground_request_count = 0;
1325 foreground_count = 0;
1326 gc_background_running = FALSE;
1327 foreground_gate = 0;
1329 if (!foreground_complete.CreateOSAutoEventNoThrow(FALSE))
1333 if (!foreground_allowed.CreateManualEventNoThrow(FALSE))
1345 void recursive_gc_sync::shutdown()
1347 if (foreground_complete.IsValid())
1348 foreground_complete.CloseEvent();
1349 if (foreground_allowed.IsValid())
1350 foreground_allowed.CloseEvent();
1353 void recursive_gc_sync::begin_background()
1355 dprintf (2, ("begin background"));
1356 foreground_request_count = 1;
1357 foreground_count = 1;
1358 foreground_allowed.Reset();
1359 gc_background_running = TRUE;
1361 void recursive_gc_sync::end_background()
1363 dprintf (2, ("end background"));
1364 gc_background_running = FALSE;
1365 foreground_gate = 1;
1366 foreground_allowed.Set();
1369 void recursive_gc_sync::begin_foreground()
1371 dprintf (2, ("begin_foreground"));
1373 bool cooperative_mode = false;
1374 if (gc_background_running)
1376 gc_heap::fire_alloc_wait_event_begin (awr_fgc_wait_for_bgc);
1377 gc_heap::alloc_wait_event_p = TRUE;
1381 Interlocked::Increment (&foreground_request_count);
1384 dprintf(2, ("Waiting sync gc point"));
1385 assert (foreground_allowed.IsValid());
1386 assert (foreground_complete.IsValid());
1388 cooperative_mode = gc_heap::enable_preemptive ();
1390 foreground_allowed.Wait(INFINITE, FALSE);
1392 dprintf(2, ("Waiting sync gc point is done"));
1394 gc_heap::disable_preemptive (cooperative_mode);
1396 if (foreground_gate)
1398 Interlocked::Increment (&foreground_count);
1399 dprintf (2, ("foreground_count: %d", (int32_t)foreground_count));
1400 if (foreground_gate)
1402 gc_heap::settings.concurrent = FALSE;
1413 goto try_again_no_inc;
1418 void recursive_gc_sync::end_foreground()
1420 dprintf (2, ("end_foreground"));
1421 if (gc_background_running)
1423 Interlocked::Decrement (&foreground_request_count);
1424 dprintf (2, ("foreground_count before decrement: %d", (int32_t)foreground_count));
1425 if (Interlocked::Decrement (&foreground_count) == 0)
1427 //c_write ((BOOL*)&foreground_gate, 0);
1428 // TODO - couldn't make the syntax work with Volatile<T>
1429 foreground_gate = 0;
1430 if (foreground_count == 0)
1432 foreground_allowed.Reset ();
1433 dprintf(2, ("setting foreground complete event"));
1434 foreground_complete.Set();
1441 BOOL recursive_gc_sync::allow_foreground()
1443 assert (gc_heap::settings.concurrent);
1444 dprintf (100, ("enter allow_foreground, f_req_count: %d, f_count: %d",
1445 (int32_t)foreground_request_count, (int32_t)foreground_count));
1447 BOOL did_fgc = FALSE;
1449 //if we have suspended the EE, just return because
1450 //some thread could be waiting on this to proceed.
1451 if (!GCHeap::GcInProgress)
1453 //TODO BACKGROUND_GC This is to stress the concurrency between
1454 //background and foreground
1455 // gc_heap::disallow_new_allocation (0);
1457 //GCToOSInterface::YieldThread(0);
1460 if (foreground_request_count != 0)
1462 //foreground wants to run
1463 //save the important settings
1464 //TODO BACKGROUND_GC be more selective about the important settings.
1465 gc_mechanisms saved_settings = gc_heap::settings;
1469 //c_write ((BOOL*)&foreground_gate, 1);
1470 // TODO - couldn't make the syntax work with Volatile<T>
1471 foreground_gate = 1;
1472 foreground_allowed.Set ();
1473 foreground_complete.Wait (INFINITE, FALSE);
1474 }while (/*foreground_request_count ||*/ foreground_gate);
1476 assert (!foreground_gate);
1478 //restore the important settings
1479 gc_heap::settings = saved_settings;
1480 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
1481 //the background GC shouldn't be using gc_high and gc_low
1482 //gc_low = lowest_address;
1483 //gc_high = highest_address;
1486 //TODO BACKGROUND_GC This is to stress the concurrency between
1487 //background and foreground
1488 // gc_heap::allow_new_allocation (0);
1492 dprintf (100, ("leave allow_foreground"));
1493 assert (gc_heap::settings.concurrent);
1497 #endif //BACKGROUND_GC
1498 #endif //DACCESS_COMPILE
1501 #if defined(COUNT_CYCLES)
1503 #pragma warning(disable:4035)
1507 unsigned GetCycleCount32() // enough for about 40 seconds
1515 #pragma warning(default:4035)
1517 #endif //COUNT_CYCLES
1520 int mark_time, plan_time, sweep_time, reloc_time, compact_time;
1523 #ifndef MULTIPLE_HEAPS
1525 #endif // MULTIPLE_HEAPS
1527 void reset_memory (uint8_t* o, size_t sizeo);
1531 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1532 static bool virtual_alloc_hardware_write_watch = false;
1533 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1535 static bool hardware_write_watch_capability = false;
1537 #ifndef DACCESS_COMPILE
1539 //check if the write watch APIs are supported.
1541 void hardware_write_watch_api_supported()
1543 if (GCToOSInterface::SupportsWriteWatch())
1545 hardware_write_watch_capability = true;
1546 dprintf (2, ("WriteWatch supported"));
1550 dprintf (2,("WriteWatch not supported"));
1554 #endif //!DACCESS_COMPILE
1556 inline bool can_use_hardware_write_watch()
1558 return hardware_write_watch_capability;
1561 inline bool can_use_write_watch_for_gc_heap()
1563 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1565 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1566 return can_use_hardware_write_watch();
1567 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1570 inline bool can_use_write_watch_for_card_table()
1572 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1575 return can_use_hardware_write_watch();
1580 #define mem_reserve (MEM_RESERVE)
1581 #endif //WRITE_WATCH
1583 //check if the low memory notification is supported
1585 #ifndef DACCESS_COMPILE
1587 void WaitLongerNoInstru (int i)
1589 // every 8th attempt:
1590 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1592 // if we're waiting for gc to finish, we should block immediately
1593 if (g_fSuspensionPending == 0)
1595 if (g_num_processors > 1)
1597 YieldProcessor(); // indicate to the processor that we are spining
1599 GCToOSInterface::YieldThread (0);
1601 GCToOSInterface::Sleep (5);
1604 GCToOSInterface::Sleep (5);
1607 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1608 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1609 // It is important that the thread is going to wait for GC. Otherwise the thread
1610 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1614 // In debug builds, all enter_spin_lock operations go through this code. If a GC has
1615 // started, it is important to block until the GC thread calls set_gc_done (since it is
1616 // guaranteed to have cleared g_TrapReturningThreads by this point). This avoids livelock
1617 // conditions which can otherwise occur if threads are allowed to spin in this function
1618 // (and therefore starve the GC thread) between the point when the GC thread sets the
1619 // WaitForGC event and the point when the GC thread clears g_TrapReturningThreads.
1620 if (gc_heap::gc_started)
1622 gc_heap::wait_for_gc_done();
1625 GCToEEInterface::DisablePreemptiveGC();
1627 else if (g_fSuspensionPending > 0)
1629 g_theGCHeap->WaitUntilGCComplete();
1634 static void safe_switch_to_thread()
1636 bool cooperative_mode = gc_heap::enable_preemptive();
1638 GCToOSInterface::YieldThread(0);
1640 gc_heap::disable_preemptive(cooperative_mode);
1644 // We need the following methods to have volatile arguments, so that they can accept
1645 // raw pointers in addition to the results of the & operator on Volatile<T>.
1648 static void enter_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1652 if (Interlocked::CompareExchange(lock, 0, -1) >= 0)
1655 while (VolatileLoad(lock) >= 0)
1657 if ((++i & 7) && !IsGCInProgress())
1659 if (g_num_processors > 1)
1661 #ifndef MULTIPLE_HEAPS
1662 int spin_count = 1024 * g_num_processors;
1663 #else //!MULTIPLE_HEAPS
1664 int spin_count = 32 * g_num_processors;
1665 #endif //!MULTIPLE_HEAPS
1666 for (int j = 0; j < spin_count; j++)
1668 if (VolatileLoad(lock) < 0 || IsGCInProgress())
1670 YieldProcessor(); // indicate to the processor that we are spining
1672 if (VolatileLoad(lock) >= 0 && !IsGCInProgress())
1674 safe_switch_to_thread();
1679 safe_switch_to_thread();
1684 WaitLongerNoInstru(i);
1692 static BOOL try_enter_spin_lock_noinstru(RAW_KEYWORD(volatile) int32_t* lock)
1694 return (Interlocked::CompareExchange(&*lock, 0, -1) < 0);
1698 static void leave_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1700 VolatileStore<int32_t>((int32_t*)lock, -1);
1706 static void enter_spin_lock(GCSpinLock *pSpinLock)
1708 enter_spin_lock_noinstru(&pSpinLock->lock);
1709 assert (pSpinLock->holding_thread == (Thread*)-1);
1710 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1714 static BOOL try_enter_spin_lock(GCSpinLock *pSpinLock)
1716 BOOL ret = try_enter_spin_lock_noinstru(&pSpinLock->lock);
1718 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1723 static void leave_spin_lock(GCSpinLock *pSpinLock)
1725 bool gc_thread_p = GCToEEInterface::WasCurrentThreadCreatedByGC();
1726 // _ASSERTE((pSpinLock->holding_thread == GCToEEInterface::GetThread()) || gc_thread_p || pSpinLock->released_by_gc_p);
1727 pSpinLock->released_by_gc_p = gc_thread_p;
1728 pSpinLock->holding_thread = (Thread*) -1;
1729 if (pSpinLock->lock != -1)
1730 leave_spin_lock_noinstru(&pSpinLock->lock);
1733 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock) \
1734 _ASSERTE((pSpinLock)->holding_thread == GCToEEInterface::GetThread());
1736 #define ASSERT_NOT_HOLDING_SPIN_LOCK(pSpinLock) \
1737 _ASSERTE((pSpinLock)->holding_thread != GCToEEInterface::GetThread());
1741 //In the concurrent version, the Enable/DisablePreemptiveGC is optional because
1742 //the gc thread call WaitLonger.
1743 void WaitLonger (int i
1744 #ifdef SYNCHRONIZATION_STATS
1745 , GCSpinLock* spin_lock
1746 #endif //SYNCHRONIZATION_STATS
1749 #ifdef SYNCHRONIZATION_STATS
1750 (spin_lock->num_wait_longer)++;
1751 #endif //SYNCHRONIZATION_STATS
1753 // every 8th attempt:
1754 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1757 // if we're waiting for gc to finish, we should block immediately
1758 if (!gc_heap::gc_started)
1760 #ifdef SYNCHRONIZATION_STATS
1761 (spin_lock->num_switch_thread_w)++;
1762 #endif //SYNCHRONIZATION_STATS
1763 if (g_num_processors > 1)
1765 YieldProcessor(); // indicate to the processor that we are spining
1767 GCToOSInterface::YieldThread (0);
1769 GCToOSInterface::Sleep (5);
1772 GCToOSInterface::Sleep (5);
1775 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1776 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1777 // It is important that the thread is going to wait for GC. Otherwise the thread
1778 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1779 if (gc_heap::gc_started)
1781 gc_heap::wait_for_gc_done();
1786 #ifdef SYNCHRONIZATION_STATS
1787 (spin_lock->num_disable_preemptive_w)++;
1788 #endif //SYNCHRONIZATION_STATS
1789 GCToEEInterface::DisablePreemptiveGC();
1794 static void enter_spin_lock (GCSpinLock* spin_lock)
1798 if (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) >= 0)
1801 while (spin_lock->lock >= 0)
1803 if ((++i & 7) && !gc_heap::gc_started)
1805 if (g_num_processors > 1)
1807 #ifndef MULTIPLE_HEAPS
1808 int spin_count = 1024 * g_num_processors;
1809 #else //!MULTIPLE_HEAPS
1810 int spin_count = 32 * g_num_processors;
1811 #endif //!MULTIPLE_HEAPS
1812 for (int j = 0; j < spin_count; j++)
1814 if (spin_lock->lock < 0 || gc_heap::gc_started)
1816 YieldProcessor(); // indicate to the processor that we are spining
1818 if (spin_lock->lock >= 0 && !gc_heap::gc_started)
1820 #ifdef SYNCHRONIZATION_STATS
1821 (spin_lock->num_switch_thread)++;
1822 #endif //SYNCHRONIZATION_STATS
1823 bool cooperative_mode = gc_heap::enable_preemptive ();
1825 GCToOSInterface::YieldThread(0);
1827 gc_heap::disable_preemptive (cooperative_mode);
1831 GCToOSInterface::YieldThread(0);
1836 #ifdef SYNCHRONIZATION_STATS
1838 #endif //SYNCHRONIZATION_STATS
1846 inline BOOL try_enter_spin_lock(GCSpinLock* spin_lock)
1848 return (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) < 0);
1852 static void leave_spin_lock (GCSpinLock * spin_lock)
1854 spin_lock->lock = -1;
1857 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock)
1861 bool gc_heap::enable_preemptive ()
1863 return GCToEEInterface::EnablePreemptiveGC();
1866 void gc_heap::disable_preemptive (bool restore_cooperative)
1868 if (restore_cooperative)
1870 GCToEEInterface::DisablePreemptiveGC();
1874 #endif // !DACCESS_COMPILE
1876 typedef void ** PTR_PTR;
1877 //This function clears a piece of memory
1878 // size has to be Dword aligned
1881 void memclr ( uint8_t* mem, size_t size)
1883 dprintf (3, ("MEMCLR: %Ix, %d", mem, size));
1884 assert ((size & (sizeof(PTR_PTR)-1)) == 0);
1885 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1888 // The compiler will recognize this pattern and replace it with memset call. We can as well just call
1889 // memset directly to make it obvious what's going on.
1890 PTR_PTR m = (PTR_PTR) mem;
1891 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
1895 memset (mem, 0, size);
1898 void memcopy (uint8_t* dmem, uint8_t* smem, size_t size)
1900 const size_t sz4ptr = sizeof(PTR_PTR)*4;
1901 const size_t sz2ptr = sizeof(PTR_PTR)*2;
1902 const size_t sz1ptr = sizeof(PTR_PTR)*1;
1904 // size must be a multiple of the pointer size
1905 assert ((size & (sizeof (PTR_PTR)-1)) == 0);
1906 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1908 // copy in groups of four pointer sized things at a time
1913 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1914 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1915 ((PTR_PTR)dmem)[2] = ((PTR_PTR)smem)[2];
1916 ((PTR_PTR)dmem)[3] = ((PTR_PTR)smem)[3];
1920 while ((size -= sz4ptr) >= sz4ptr);
1923 // still two pointer sized things or more left to copy?
1926 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1927 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1932 // still one pointer sized thing left to copy?
1935 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1943 ptrdiff_t round_down (ptrdiff_t add, int pitch)
1945 return ((add / pitch) * pitch);
1948 #if defined(FEATURE_STRUCTALIGN) && defined(RESPECT_LARGE_ALIGNMENT)
1949 // FEATURE_STRUCTALIGN allows the compiler to dictate the alignment,
1950 // i.e, if a larger alignment matters or is beneficial, the compiler
1951 // generated info tells us so. RESPECT_LARGE_ALIGNMENT is just the
1952 // converse - it's a heuristic for the GC to use a larger alignment.
1953 #error FEATURE_STRUCTALIGN should imply !RESPECT_LARGE_ALIGNMENT
1956 #if defined(FEATURE_STRUCTALIGN) && defined(FEATURE_LOH_COMPACTION)
1957 #error FEATURE_STRUCTALIGN and FEATURE_LOH_COMPACTION are mutually exclusive
1960 #if defined(GROWABLE_SEG_MAPPING_TABLE) && !defined(SEG_MAPPING_TABLE)
1961 #error if GROWABLE_SEG_MAPPING_TABLE is defined, SEG_MAPPING_TABLE must be defined
1965 BOOL same_large_alignment_p (uint8_t* p1, uint8_t* p2)
1967 #ifdef RESPECT_LARGE_ALIGNMENT
1968 return ((((size_t)p1 ^ (size_t)p2) & 7) == 0);
1970 UNREFERENCED_PARAMETER(p1);
1971 UNREFERENCED_PARAMETER(p2);
1973 #endif //RESPECT_LARGE_ALIGNMENT
1977 size_t switch_alignment_size (BOOL already_padded_p)
1979 if (already_padded_p)
1980 return DATA_ALIGNMENT;
1982 return (Align (min_obj_size) +((Align (min_obj_size)&DATA_ALIGNMENT)^DATA_ALIGNMENT));
1986 #ifdef FEATURE_STRUCTALIGN
1987 void set_node_aligninfo (uint8_t *node, int requiredAlignment, ptrdiff_t pad);
1988 void clear_node_aligninfo (uint8_t *node);
1989 #else // FEATURE_STRUCTALIGN
1990 #define node_realigned(node) (((plug_and_reloc*)(node))[-1].reloc & 1)
1991 void set_node_realigned (uint8_t* node);
1992 void clear_node_realigned(uint8_t* node);
1993 #endif // FEATURE_STRUCTALIGN
1996 size_t AlignQword (size_t nbytes)
1998 #ifdef FEATURE_STRUCTALIGN
1999 // This function is used to align everything on the large object
2000 // heap to an 8-byte boundary, to reduce the number of unaligned
2001 // accesses to (say) arrays of doubles. With FEATURE_STRUCTALIGN,
2002 // the compiler dictates the optimal alignment instead of having
2003 // a heuristic in the GC.
2004 return Align (nbytes);
2005 #else // FEATURE_STRUCTALIGN
2006 return (nbytes + 7) & ~7;
2007 #endif // FEATURE_STRUCTALIGN
2011 BOOL Aligned (size_t n)
2013 return (n & ALIGNCONST) == 0;
2016 #define OBJECT_ALIGNMENT_OFFSET (sizeof(MethodTable *))
2018 #ifdef FEATURE_STRUCTALIGN
2019 #define MAX_STRUCTALIGN OS_PAGE_SIZE
2020 #else // FEATURE_STRUCTALIGN
2021 #define MAX_STRUCTALIGN 0
2022 #endif // FEATURE_STRUCTALIGN
2024 #ifdef FEATURE_STRUCTALIGN
2026 ptrdiff_t AdjustmentForMinPadSize(ptrdiff_t pad, int requiredAlignment)
2028 // The resulting alignpad must be either 0 or at least min_obj_size.
2029 // Note that by computing the following difference on unsigned types,
2030 // we can do the range check 0 < alignpad < min_obj_size with a
2031 // single conditional branch.
2032 if ((size_t)(pad - DATA_ALIGNMENT) < Align (min_obj_size) - DATA_ALIGNMENT)
2034 return requiredAlignment;
2040 uint8_t* StructAlign (uint8_t* origPtr, int requiredAlignment, ptrdiff_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2042 // required alignment must be a power of two
2043 _ASSERTE(((size_t)origPtr & ALIGNCONST) == 0);
2044 _ASSERTE(((requiredAlignment - 1) & requiredAlignment) == 0);
2045 _ASSERTE(requiredAlignment >= sizeof(void *));
2046 _ASSERTE(requiredAlignment <= MAX_STRUCTALIGN);
2048 // When this method is invoked for individual objects (i.e., alignmentOffset
2049 // is just the size of the PostHeader), what needs to be aligned when
2050 // we're done is the pointer to the payload of the object (which means
2051 // the actual resulting object pointer is typically not aligned).
2053 uint8_t* result = (uint8_t*)Align ((size_t)origPtr + alignmentOffset, requiredAlignment-1) - alignmentOffset;
2054 ptrdiff_t alignpad = result - origPtr;
2056 return result + AdjustmentForMinPadSize (alignpad, requiredAlignment);
2060 ptrdiff_t ComputeStructAlignPad (uint8_t* plug, int requiredAlignment, size_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2062 return StructAlign (plug, requiredAlignment, alignmentOffset) - plug;
2065 BOOL IsStructAligned (uint8_t *ptr, int requiredAlignment)
2067 return StructAlign (ptr, requiredAlignment) == ptr;
2071 ptrdiff_t ComputeMaxStructAlignPad (int requiredAlignment)
2073 if (requiredAlignment == DATA_ALIGNMENT)
2075 // Since a non-zero alignment padding cannot be less than min_obj_size (so we can fit the
2076 // alignment padding object), the worst-case alignment padding is correspondingly larger
2077 // than the required alignment.
2078 return requiredAlignment + Align (min_obj_size) - DATA_ALIGNMENT;
2082 ptrdiff_t ComputeMaxStructAlignPadLarge (int requiredAlignment)
2084 if (requiredAlignment <= get_alignment_constant (TRUE)+1)
2086 // This is the same as ComputeMaxStructAlignPad, except that in addition to leaving space
2087 // for padding before the actual object, it also leaves space for filling a gap after the
2088 // actual object. This is needed on the large object heap, as the outer allocation functions
2089 // don't operate on an allocation context (which would have left space for the final gap).
2090 return requiredAlignment + Align (min_obj_size) * 2 - DATA_ALIGNMENT;
2093 uint8_t* gc_heap::pad_for_alignment (uint8_t* newAlloc, int requiredAlignment, size_t size, alloc_context* acontext)
2095 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2096 if (alignedPtr != newAlloc) {
2097 make_unused_array (newAlloc, alignedPtr - newAlloc);
2099 acontext->alloc_ptr = alignedPtr + Align (size);
2103 uint8_t* gc_heap::pad_for_alignment_large (uint8_t* newAlloc, int requiredAlignment, size_t size)
2105 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2106 if (alignedPtr != newAlloc) {
2107 make_unused_array (newAlloc, alignedPtr - newAlloc);
2109 if (alignedPtr < newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment)) {
2110 make_unused_array (alignedPtr + AlignQword (size), newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment) - alignedPtr);
2114 #else // FEATURE_STRUCTALIGN
2115 #define ComputeMaxStructAlignPad(requiredAlignment) 0
2116 #define ComputeMaxStructAlignPadLarge(requiredAlignment) 0
2117 #endif // FEATURE_STRUCTALIGN
2119 //CLR_SIZE is the max amount of bytes from gen0 that is set to 0 in one chunk
2121 #define CLR_SIZE ((size_t)(8*1024))
2123 #define CLR_SIZE ((size_t)(8*1024))
2126 #define END_SPACE_AFTER_GC (LARGE_OBJECT_SIZE + MAX_STRUCTALIGN)
2128 #ifdef BACKGROUND_GC
2129 #define SEGMENT_INITIAL_COMMIT (2*OS_PAGE_SIZE)
2131 #define SEGMENT_INITIAL_COMMIT (OS_PAGE_SIZE)
2132 #endif //BACKGROUND_GC
2138 #define INITIAL_ALLOC ((size_t)((size_t)4*1024*1024*1024))
2139 #define LHEAP_ALLOC ((size_t)(1024*1024*256))
2143 #define INITIAL_ALLOC ((size_t)(1024*1024*64))
2144 #define LHEAP_ALLOC ((size_t)(1024*1024*32))
2152 #define INITIAL_ALLOC ((size_t)(1024*1024*256))
2153 #define LHEAP_ALLOC ((size_t)(1024*1024*128))
2157 #define INITIAL_ALLOC ((size_t)(1024*1024*16))
2158 #define LHEAP_ALLOC ((size_t)(1024*1024*16))
2164 //amount in bytes of the etw allocation tick
2165 const size_t etw_allocation_tick = 100*1024;
2167 const size_t low_latency_alloc = 256*1024;
2169 const size_t fgn_check_quantum = 2*1024*1024;
2172 const int max_snoop_level = 128;
2177 //threshold of heap size to turn on card bundles.
2178 #define SH_TH_CARD_BUNDLE (40*1024*1024)
2179 #define MH_TH_CARD_BUNDLE (180*1024*1024)
2180 #endif //CARD_BUNDLE
2182 #define GC_EPHEMERAL_DECOMMIT_TIMEOUT 5000
2185 size_t align_on_page (size_t add)
2187 return ((add + OS_PAGE_SIZE - 1) & ~((size_t)OS_PAGE_SIZE - 1));
2191 uint8_t* align_on_page (uint8_t* add)
2193 return (uint8_t*)align_on_page ((size_t) add);
2197 size_t align_lower_page (size_t add)
2199 return (add & ~((size_t)OS_PAGE_SIZE - 1));
2203 uint8_t* align_lower_page (uint8_t* add)
2205 return (uint8_t*)align_lower_page ((size_t)add);
2209 size_t align_write_watch_lower_page (size_t add)
2211 return (add & ~(WRITE_WATCH_UNIT_SIZE - 1));
2215 uint8_t* align_write_watch_lower_page (uint8_t* add)
2217 return (uint8_t*)align_lower_page ((size_t)add);
2222 BOOL power_of_two_p (size_t integer)
2224 return !(integer & (integer-1));
2228 BOOL oddp (size_t integer)
2230 return (integer & 1) != 0;
2233 // we only ever use this for WORDs.
2234 size_t logcount (size_t word)
2236 //counts the number of high bits in a 16 bit word.
2237 assert (word < 0x10000);
2239 count = (word & 0x5555) + ( (word >> 1 ) & 0x5555);
2240 count = (count & 0x3333) + ( (count >> 2) & 0x3333);
2241 count = (count & 0x0F0F) + ( (count >> 4) & 0x0F0F);
2242 count = (count & 0x00FF) + ( (count >> 8) & 0x00FF);
2246 #ifndef DACCESS_COMPILE
2248 void stomp_write_barrier_resize(bool is_runtime_suspended, bool requires_upper_bounds_check)
2250 WriteBarrierParameters args = {};
2251 args.operation = WriteBarrierOp::StompResize;
2252 args.is_runtime_suspended = is_runtime_suspended;
2253 args.requires_upper_bounds_check = requires_upper_bounds_check;
2255 args.card_table = g_gc_card_table;
2256 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2257 args.card_bundle_table = g_gc_card_bundle_table;
2260 args.lowest_address = g_gc_lowest_address;
2261 args.highest_address = g_gc_highest_address;
2263 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2264 if (SoftwareWriteWatch::IsEnabledForGCHeap())
2266 args.write_watch_table = g_gc_sw_ww_table;
2268 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2270 GCToEEInterface::StompWriteBarrier(&args);
2273 void stomp_write_barrier_ephemeral(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2275 WriteBarrierParameters args = {};
2276 args.operation = WriteBarrierOp::StompEphemeral;
2277 args.is_runtime_suspended = true;
2278 args.ephemeral_low = ephemeral_low;
2279 args.ephemeral_high = ephemeral_high;
2280 GCToEEInterface::StompWriteBarrier(&args);
2283 void stomp_write_barrier_initialize(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2285 WriteBarrierParameters args = {};
2286 args.operation = WriteBarrierOp::Initialize;
2287 args.is_runtime_suspended = true;
2288 args.requires_upper_bounds_check = false;
2289 args.card_table = g_gc_card_table;
2291 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2292 args.card_bundle_table = g_gc_card_bundle_table;
2295 args.lowest_address = g_gc_lowest_address;
2296 args.highest_address = g_gc_highest_address;
2297 args.ephemeral_low = ephemeral_low;
2298 args.ephemeral_high = ephemeral_high;
2299 GCToEEInterface::StompWriteBarrier(&args);
2302 #endif // DACCESS_COMPILE
2304 //extract the low bits [0,low[ of a uint32_t
2305 #define lowbits(wrd, bits) ((wrd) & ((1 << (bits))-1))
2306 //extract the high bits [high, 32] of a uint32_t
2307 #define highbits(wrd, bits) ((wrd) & ~((1 << (bits))-1))
2309 // Things we need to manually initialize:
2310 // gen0 min_size - based on cache
2311 // gen0/1 max_size - based on segment size
2312 static static_data static_data_table[latency_level_last - latency_level_first + 1][NUMBERGENERATIONS] =
2314 // latency_level_memory_footprint
2317 {0, 0, 40000, 0.5f, 9.0f, 20.0f, 1000, 1},
2319 {163840, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2321 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2323 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2326 // latency_level_balanced
2330 #ifdef MULTIPLE_HEAPS
2334 #endif //MULTIPLE_HEAPS
2337 {9*32*1024, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2339 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2341 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2348 class CObjectHeader;
2352 class c_synchronize;
2354 #ifdef FEATURE_PREMORTEM_FINALIZATION
2355 #ifndef DACCESS_COMPILE
2357 HRESULT AllocateCFinalize(CFinalize **pCFinalize);
2358 #endif //!DACCESS_COMPILE
2359 #endif // FEATURE_PREMORTEM_FINALIZATION
2361 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address);
2364 #ifdef USE_INTROSORT
2365 #define _sort introsort::sort
2366 #else //USE_INTROSORT
2367 #define _sort qsort1
2368 void qsort1(uint8_t** low, uint8_t** high, unsigned int depth);
2369 #endif //USE_INTROSORT
2371 void* virtual_alloc (size_t size);
2372 void virtual_free (void* add, size_t size);
2374 /* per heap static initialization */
2376 #ifndef MULTIPLE_HEAPS
2377 uint32_t* gc_heap::mark_array;
2378 #endif //MULTIPLE_HEAPS
2382 uint8_t** gc_heap::g_mark_list;
2384 #ifdef PARALLEL_MARK_LIST_SORT
2385 uint8_t** gc_heap::g_mark_list_copy;
2386 #endif //PARALLEL_MARK_LIST_SORT
2388 size_t gc_heap::mark_list_size;
2391 #ifdef SEG_MAPPING_TABLE
2392 seg_mapping* seg_mapping_table;
2393 #endif //SEG_MAPPING_TABLE
2395 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
2396 sorted_table* gc_heap::seg_table;
2397 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
2399 #ifdef MULTIPLE_HEAPS
2400 GCEvent gc_heap::ee_suspend_event;
2401 size_t gc_heap::min_balance_threshold = 0;
2402 #endif //MULTIPLE_HEAPS
2404 VOLATILE(BOOL) gc_heap::gc_started;
2406 #ifdef MULTIPLE_HEAPS
2408 GCEvent gc_heap::gc_start_event;
2410 bool gc_heap::gc_thread_no_affinitize_p = false;
2412 int gc_heap::n_heaps;
2414 gc_heap** gc_heap::g_heaps;
2416 size_t* gc_heap::g_promoted;
2419 int* gc_heap::g_mark_stack_busy;
2423 #ifdef BACKGROUND_GC
2424 size_t* gc_heap::g_bpromoted;
2425 #endif //BACKGROUND_GC
2427 #else //MULTIPLE_HEAPS
2429 size_t gc_heap::g_promoted;
2431 #ifdef BACKGROUND_GC
2432 size_t gc_heap::g_bpromoted;
2433 #endif //BACKGROUND_GC
2435 #endif //MULTIPLE_HEAPS
2437 size_t gc_heap::reserved_memory = 0;
2438 size_t gc_heap::reserved_memory_limit = 0;
2439 BOOL gc_heap::g_low_memory_status;
2441 #ifndef DACCESS_COMPILE
2442 static gc_reason gc_trigger_reason = reason_empty;
2443 #endif //DACCESS_COMPILE
2445 gc_latency_level gc_heap::latency_level = latency_level_default;
2447 gc_mechanisms gc_heap::settings;
2449 gc_history_global gc_heap::gc_data_global;
2451 size_t gc_heap::gc_last_ephemeral_decommit_time = 0;
2453 size_t gc_heap::gc_gen0_desired_high;
2456 double gc_heap::short_plugs_pad_ratio = 0;
2457 #endif //SHORT_PLUGS
2460 #define MAX_ALLOWED_MEM_LOAD 85
2462 // consider putting this in dynamic data -
2463 // we may want different values for workstation
2465 #define MIN_YOUNGEST_GEN_DESIRED (16*1024*1024)
2467 size_t gc_heap::youngest_gen_desired_th;
2470 uint32_t gc_heap::last_gc_memory_load = 0;
2472 size_t gc_heap::last_gc_heap_size = 0;
2474 size_t gc_heap::last_gc_fragmentation = 0;
2476 uint64_t gc_heap::mem_one_percent = 0;
2478 uint32_t gc_heap::high_memory_load_th = 0;
2480 uint64_t gc_heap::total_physical_mem = 0;
2482 uint64_t gc_heap::entry_available_physical_mem = 0;
2484 #ifdef BACKGROUND_GC
2485 GCEvent gc_heap::bgc_start_event;
2487 gc_mechanisms gc_heap::saved_bgc_settings;
2489 GCEvent gc_heap::background_gc_done_event;
2491 GCEvent gc_heap::ee_proceed_event;
2493 bool gc_heap::gc_can_use_concurrent = false;
2495 bool gc_heap::temp_disable_concurrent_p = false;
2497 uint32_t gc_heap::cm_in_progress = FALSE;
2499 BOOL gc_heap::dont_restart_ee_p = FALSE;
2501 BOOL gc_heap::keep_bgc_threads_p = FALSE;
2503 GCEvent gc_heap::bgc_threads_sync_event;
2505 BOOL gc_heap::do_ephemeral_gc_p = FALSE;
2507 BOOL gc_heap::do_concurrent_p = FALSE;
2509 size_t gc_heap::ephemeral_fgc_counts[max_generation];
2511 BOOL gc_heap::alloc_wait_event_p = FALSE;
2513 VOLATILE(c_gc_state) gc_heap::current_c_gc_state = c_gc_state_free;
2515 #endif //BACKGROUND_GC
2517 #ifndef MULTIPLE_HEAPS
2518 #ifdef SPINLOCK_HISTORY
2519 int gc_heap::spinlock_info_index = 0;
2520 spinlock_info gc_heap::last_spinlock_info[max_saved_spinlock_info + 8];
2521 #endif //SPINLOCK_HISTORY
2523 size_t gc_heap::fgn_last_alloc = 0;
2525 int gc_heap::generation_skip_ratio = 100;
2527 uint64_t gc_heap::loh_alloc_since_cg = 0;
2529 BOOL gc_heap::elevation_requested = FALSE;
2531 BOOL gc_heap::last_gc_before_oom = FALSE;
2533 #ifdef BACKGROUND_GC
2534 uint8_t* gc_heap::background_saved_lowest_address = 0;
2535 uint8_t* gc_heap::background_saved_highest_address = 0;
2536 uint8_t* gc_heap::next_sweep_obj = 0;
2537 uint8_t* gc_heap::current_sweep_pos = 0;
2538 exclusive_sync* gc_heap::bgc_alloc_lock;
2539 #endif //BACKGROUND_GC
2541 oom_history gc_heap::oom_info;
2543 fgm_history gc_heap::fgm_result;
2545 BOOL gc_heap::ro_segments_in_range;
2547 size_t gc_heap::gen0_big_free_spaces = 0;
2549 uint8_t* gc_heap::ephemeral_low;
2551 uint8_t* gc_heap::ephemeral_high;
2553 uint8_t* gc_heap::lowest_address;
2555 uint8_t* gc_heap::highest_address;
2557 BOOL gc_heap::ephemeral_promotion;
2559 uint8_t* gc_heap::saved_ephemeral_plan_start[NUMBERGENERATIONS-1];
2560 size_t gc_heap::saved_ephemeral_plan_start_size[NUMBERGENERATIONS-1];
2562 short* gc_heap::brick_table;
2564 uint32_t* gc_heap::card_table;
2567 uint32_t* gc_heap::card_bundle_table;
2568 #endif //CARD_BUNDLE
2570 uint8_t* gc_heap::gc_low;
2572 uint8_t* gc_heap::gc_high;
2574 uint8_t* gc_heap::demotion_low;
2576 uint8_t* gc_heap::demotion_high;
2578 BOOL gc_heap::demote_gen1_p = TRUE;
2580 uint8_t* gc_heap::last_gen1_pin_end;
2582 gen_to_condemn_tuning gc_heap::gen_to_condemn_reasons;
2584 size_t gc_heap::etw_allocation_running_amount[2];
2586 int gc_heap::gc_policy = 0;
2588 size_t gc_heap::allocation_running_time;
2590 size_t gc_heap::allocation_running_amount;
2592 heap_segment* gc_heap::ephemeral_heap_segment = 0;
2594 BOOL gc_heap::blocking_collection = FALSE;
2596 heap_segment* gc_heap::freeable_large_heap_segment = 0;
2598 size_t gc_heap::time_bgc_last = 0;
2600 size_t gc_heap::mark_stack_tos = 0;
2602 size_t gc_heap::mark_stack_bos = 0;
2604 size_t gc_heap::mark_stack_array_length = 0;
2606 mark* gc_heap::mark_stack_array = 0;
2608 BOOL gc_heap::verify_pinned_queue_p = FALSE;
2610 uint8_t* gc_heap::oldest_pinned_plug = 0;
2612 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
2613 size_t gc_heap::num_pinned_objects = 0;
2614 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
2616 #ifdef FEATURE_LOH_COMPACTION
2617 size_t gc_heap::loh_pinned_queue_tos = 0;
2619 size_t gc_heap::loh_pinned_queue_bos = 0;
2621 size_t gc_heap::loh_pinned_queue_length = 0;
2623 mark* gc_heap::loh_pinned_queue = 0;
2625 BOOL gc_heap::loh_compacted_p = FALSE;
2626 #endif //FEATURE_LOH_COMPACTION
2628 #ifdef BACKGROUND_GC
2630 EEThreadId gc_heap::bgc_thread_id;
2632 uint8_t* gc_heap::background_written_addresses [array_size+2];
2634 heap_segment* gc_heap::freeable_small_heap_segment = 0;
2636 size_t gc_heap::bgc_overflow_count = 0;
2638 size_t gc_heap::bgc_begin_loh_size = 0;
2639 size_t gc_heap::end_loh_size = 0;
2641 uint32_t gc_heap::bgc_alloc_spin_loh = 0;
2643 size_t gc_heap::bgc_loh_size_increased = 0;
2645 size_t gc_heap::bgc_loh_allocated_in_free = 0;
2647 size_t gc_heap::background_soh_alloc_count = 0;
2649 size_t gc_heap::background_loh_alloc_count = 0;
2651 uint8_t** gc_heap::background_mark_stack_tos = 0;
2653 uint8_t** gc_heap::background_mark_stack_array = 0;
2655 size_t gc_heap::background_mark_stack_array_length = 0;
2657 uint8_t* gc_heap::background_min_overflow_address =0;
2659 uint8_t* gc_heap::background_max_overflow_address =0;
2661 BOOL gc_heap::processed_soh_overflow_p = FALSE;
2663 uint8_t* gc_heap::background_min_soh_overflow_address =0;
2665 uint8_t* gc_heap::background_max_soh_overflow_address =0;
2667 heap_segment* gc_heap::saved_sweep_ephemeral_seg = 0;
2669 uint8_t* gc_heap::saved_sweep_ephemeral_start = 0;
2671 heap_segment* gc_heap::saved_overflow_ephemeral_seg = 0;
2673 Thread* gc_heap::bgc_thread = 0;
2675 BOOL gc_heap::expanded_in_fgc = FALSE;
2677 uint8_t** gc_heap::c_mark_list = 0;
2679 size_t gc_heap::c_mark_list_length = 0;
2681 size_t gc_heap::c_mark_list_index = 0;
2683 gc_history_per_heap gc_heap::bgc_data_per_heap;
2685 BOOL gc_heap::bgc_thread_running;
2687 CLRCriticalSection gc_heap::bgc_threads_timeout_cs;
2689 GCEvent gc_heap::gc_lh_block_event;
2691 #endif //BACKGROUND_GC
2694 uint8_t** gc_heap::mark_list;
2695 uint8_t** gc_heap::mark_list_index;
2696 uint8_t** gc_heap::mark_list_end;
2700 snoop_stats_data gc_heap::snoop_stat;
2701 #endif //SNOOP_STATS
2703 uint8_t* gc_heap::min_overflow_address = MAX_PTR;
2705 uint8_t* gc_heap::max_overflow_address = 0;
2707 uint8_t* gc_heap::shigh = 0;
2709 uint8_t* gc_heap::slow = MAX_PTR;
2711 size_t gc_heap::ordered_free_space_indices[MAX_NUM_BUCKETS];
2713 size_t gc_heap::saved_ordered_free_space_indices[MAX_NUM_BUCKETS];
2715 size_t gc_heap::ordered_plug_indices[MAX_NUM_BUCKETS];
2717 size_t gc_heap::saved_ordered_plug_indices[MAX_NUM_BUCKETS];
2719 BOOL gc_heap::ordered_plug_indices_init = FALSE;
2721 BOOL gc_heap::use_bestfit = FALSE;
2723 uint8_t* gc_heap::bestfit_first_pin = 0;
2725 BOOL gc_heap::commit_end_of_seg = FALSE;
2727 size_t gc_heap::max_free_space_items = 0;
2729 size_t gc_heap::free_space_buckets = 0;
2731 size_t gc_heap::free_space_items = 0;
2733 int gc_heap::trimmed_free_space_index = 0;
2735 size_t gc_heap::total_ephemeral_plugs = 0;
2737 seg_free_spaces* gc_heap::bestfit_seg = 0;
2739 size_t gc_heap::total_ephemeral_size = 0;
2743 size_t gc_heap::internal_root_array_length = initial_internal_roots;
2745 uint8_t** gc_heap::internal_root_array = 0;
2747 size_t gc_heap::internal_root_array_index = 0;
2749 BOOL gc_heap::heap_analyze_success = TRUE;
2751 uint8_t* gc_heap::current_obj = 0;
2752 size_t gc_heap::current_obj_size = 0;
2754 #endif //HEAP_ANALYZE
2756 #ifdef GC_CONFIG_DRIVEN
2757 size_t gc_heap::interesting_data_per_gc[max_idp_count];
2758 //size_t gc_heap::interesting_data_per_heap[max_idp_count];
2759 //size_t gc_heap::interesting_mechanisms_per_heap[max_im_count];
2760 #endif //GC_CONFIG_DRIVEN
2761 #endif //MULTIPLE_HEAPS
2763 no_gc_region_info gc_heap::current_no_gc_region_info;
2764 BOOL gc_heap::proceed_with_gc_p = FALSE;
2765 GCSpinLock gc_heap::gc_lock;
2767 size_t gc_heap::eph_gen_starts_size = 0;
2768 heap_segment* gc_heap::segment_standby_list;
2769 size_t gc_heap::last_gc_index = 0;
2770 #ifdef SEG_MAPPING_TABLE
2771 size_t gc_heap::min_segment_size = 0;
2772 size_t gc_heap::min_segment_size_shr = 0;
2773 #endif //SEG_MAPPING_TABLE
2774 size_t gc_heap::soh_segment_size = 0;
2775 size_t gc_heap::min_loh_segment_size = 0;
2776 size_t gc_heap::segment_info_size = 0;
2778 #ifdef GC_CONFIG_DRIVEN
2779 size_t gc_heap::time_init = 0;
2780 size_t gc_heap::time_since_init = 0;
2781 size_t gc_heap::compact_or_sweep_gcs[2];
2782 #endif //GC_CONFIG_DRIVEN
2784 #ifdef FEATURE_LOH_COMPACTION
2785 BOOL gc_heap::loh_compaction_always_p = FALSE;
2786 gc_loh_compaction_mode gc_heap::loh_compaction_mode = loh_compaction_default;
2787 int gc_heap::loh_pinned_queue_decay = LOH_PIN_DECAY;
2789 #endif //FEATURE_LOH_COMPACTION
2791 GCEvent gc_heap::full_gc_approach_event;
2793 GCEvent gc_heap::full_gc_end_event;
2795 uint32_t gc_heap::fgn_maxgen_percent = 0;
2797 uint32_t gc_heap::fgn_loh_percent = 0;
2799 #ifdef BACKGROUND_GC
2800 BOOL gc_heap::fgn_last_gc_was_concurrent = FALSE;
2801 #endif //BACKGROUND_GC
2803 VOLATILE(bool) gc_heap::full_gc_approach_event_set;
2805 size_t gc_heap::full_gc_counts[gc_type_max];
2807 BOOL gc_heap::should_expand_in_full_gc = FALSE;
2810 BOOL gc_heap::heap_analyze_enabled = FALSE;
2811 #endif //HEAP_ANALYZE
2813 #ifndef MULTIPLE_HEAPS
2815 alloc_list gc_heap::loh_alloc_list [NUM_LOH_ALIST-1];
2816 alloc_list gc_heap::gen2_alloc_list[NUM_GEN2_ALIST-1];
2818 dynamic_data gc_heap::dynamic_data_table [NUMBERGENERATIONS+1];
2819 gc_history_per_heap gc_heap::gc_data_per_heap;
2820 size_t gc_heap::maxgen_pinned_compact_before_advance = 0;
2822 uint8_t* gc_heap::alloc_allocated = 0;
2824 size_t gc_heap::allocation_quantum = CLR_SIZE;
2826 GCSpinLock gc_heap::more_space_lock;
2828 #ifdef SYNCHRONIZATION_STATS
2829 unsigned int gc_heap::good_suspension = 0;
2830 unsigned int gc_heap::bad_suspension = 0;
2831 uint64_t gc_heap::total_msl_acquire = 0;
2832 unsigned int gc_heap::num_msl_acquired = 0;
2833 unsigned int gc_heap::num_high_msl_acquire = 0;
2834 unsigned int gc_heap::num_low_msl_acquire = 0;
2835 #endif //SYNCHRONIZATION_STATS
2837 size_t gc_heap::alloc_contexts_used = 0;
2838 size_t gc_heap::soh_allocation_no_gc = 0;
2839 size_t gc_heap::loh_allocation_no_gc = 0;
2840 bool gc_heap::no_gc_oom_p = false;
2841 heap_segment* gc_heap::saved_loh_segment_no_gc = 0;
2843 #endif //MULTIPLE_HEAPS
2845 #ifndef MULTIPLE_HEAPS
2847 BOOL gc_heap::gen0_bricks_cleared = FALSE;
2850 int gc_heap::gen0_must_clear_bricks = 0;
2851 #endif //FFIND_OBJECT
2853 #ifdef FEATURE_PREMORTEM_FINALIZATION
2854 CFinalize* gc_heap::finalize_queue = 0;
2855 #endif // FEATURE_PREMORTEM_FINALIZATION
2857 generation gc_heap::generation_table [NUMBERGENERATIONS + 1];
2859 size_t gc_heap::interesting_data_per_heap[max_idp_count];
2861 size_t gc_heap::compact_reasons_per_heap[max_compact_reasons_count];
2863 size_t gc_heap::expand_mechanisms_per_heap[max_expand_mechanisms_count];
2865 size_t gc_heap::interesting_mechanism_bits_per_heap[max_gc_mechanism_bits_count];
2867 #endif // MULTIPLE_HEAPS
2869 /* end of per heap static initialization */
2871 /* end of static initialization */
2873 #ifndef DACCESS_COMPILE
2875 void gen_to_condemn_tuning::print (int heap_num)
2878 dprintf (DT_LOG_0, ("condemned reasons (%d %d)", condemn_reasons_gen, condemn_reasons_condition));
2879 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_gen_header));
2880 gc_condemn_reason_gen r_gen;
2881 for (int i = 0; i < gcrg_max; i++)
2883 r_gen = (gc_condemn_reason_gen)(i);
2884 str_reasons_gen[i * 2] = get_gen_char (get_gen (r_gen));
2886 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_gen));
2888 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_condition_header));
2889 gc_condemn_reason_condition r_condition;
2890 for (int i = 0; i < gcrc_max; i++)
2892 r_condition = (gc_condemn_reason_condition)(i);
2893 str_reasons_condition[i * 2] = get_condition_char (get_condition (r_condition));
2896 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_condition));
2898 UNREFERENCED_PARAMETER(heap_num);
2902 void gc_generation_data::print (int heap_num, int gen_num)
2904 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2905 dprintf (DT_LOG_0, ("[%2d]gen%d beg %Id fl %Id fo %Id end %Id fl %Id fo %Id in %Id p %Id np %Id alloc %Id",
2908 free_list_space_before, free_obj_space_before,
2910 free_list_space_after, free_obj_space_after,
2911 in, pinned_surv, npinned_surv,
2914 UNREFERENCED_PARAMETER(heap_num);
2915 UNREFERENCED_PARAMETER(gen_num);
2916 #endif //SIMPLE_DPRINTF && DT_LOG
2919 void gc_history_per_heap::set_mechanism (gc_mechanism_per_heap mechanism_per_heap, uint32_t value)
2921 uint32_t* mechanism = &mechanisms[mechanism_per_heap];
2923 *mechanism |= mechanism_mask;
2924 *mechanism |= (1 << value);
2927 gc_mechanism_descr* descr = &gc_mechanisms_descr[mechanism_per_heap];
2928 dprintf (DT_LOG_0, ("setting %s: %s",
2930 (descr->descr)[value]));
2934 void gc_history_per_heap::print()
2936 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2937 for (int i = 0; i < (sizeof (gen_data)/sizeof (gc_generation_data)); i++)
2939 gen_data[i].print (heap_index, i);
2942 dprintf (DT_LOG_0, ("fla %Id flr %Id esa %Id ca %Id pa %Id paa %Id, rfle %d, ec %Id",
2943 maxgen_size_info.free_list_allocated,
2944 maxgen_size_info.free_list_rejected,
2945 maxgen_size_info.end_seg_allocated,
2946 maxgen_size_info.condemned_allocated,
2947 maxgen_size_info.pinned_allocated,
2948 maxgen_size_info.pinned_allocated_advance,
2949 maxgen_size_info.running_free_list_efficiency,
2950 extra_gen0_committed));
2953 gc_mechanism_descr* descr = 0;
2955 for (int i = 0; i < max_mechanism_per_heap; i++)
2957 mechanism = get_mechanism ((gc_mechanism_per_heap)i);
2961 descr = &gc_mechanisms_descr[(gc_mechanism_per_heap)i];
2962 dprintf (DT_LOG_0, ("[%2d]%s%s",
2965 (descr->descr)[mechanism]));
2968 #endif //SIMPLE_DPRINTF && DT_LOG
2971 void gc_history_global::print()
2974 char str_settings[64];
2975 memset (str_settings, '|', sizeof (char) * 64);
2976 str_settings[max_global_mechanisms_count*2] = 0;
2978 for (int i = 0; i < max_global_mechanisms_count; i++)
2980 str_settings[i * 2] = (get_mechanism_p ((gc_global_mechanism_p)i) ? 'Y' : 'N');
2983 dprintf (DT_LOG_0, ("[hp]|c|p|o|d|b|e|"));
2985 dprintf (DT_LOG_0, ("%4d|%s", num_heaps, str_settings));
2986 dprintf (DT_LOG_0, ("Condemned gen%d(reason: %s; mode: %s), youngest budget %Id(%d), memload %d",
2987 condemned_generation,
2988 str_gc_reasons[reason],
2989 str_gc_pause_modes[pause_mode],
2990 final_youngest_desired,
2991 gen0_reduction_count,
2996 void gc_heap::fire_per_heap_hist_event (gc_history_per_heap* current_gc_data_per_heap, int heap_num)
2998 maxgen_size_increase* maxgen_size_info = &(current_gc_data_per_heap->maxgen_size_info);
2999 FIRE_EVENT(GCPerHeapHistory_V3,
3000 (void *)(maxgen_size_info->free_list_allocated),
3001 (void *)(maxgen_size_info->free_list_rejected),
3002 (void *)(maxgen_size_info->end_seg_allocated),
3003 (void *)(maxgen_size_info->condemned_allocated),
3004 (void *)(maxgen_size_info->pinned_allocated),
3005 (void *)(maxgen_size_info->pinned_allocated_advance),
3006 maxgen_size_info->running_free_list_efficiency,
3007 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons0(),
3008 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons1(),
3009 current_gc_data_per_heap->mechanisms[gc_heap_compact],
3010 current_gc_data_per_heap->mechanisms[gc_heap_expand],
3011 current_gc_data_per_heap->heap_index,
3012 (void *)(current_gc_data_per_heap->extra_gen0_committed),
3013 (max_generation + 2),
3014 (uint32_t)(sizeof (gc_generation_data)),
3015 (void *)&(current_gc_data_per_heap->gen_data[0]));
3017 current_gc_data_per_heap->print();
3018 current_gc_data_per_heap->gen_to_condemn_reasons.print (heap_num);
3021 void gc_heap::fire_pevents()
3024 settings.record (&gc_data_global);
3025 gc_data_global.print();
3027 FIRE_EVENT(GCGlobalHeapHistory_V2, gc_data_global.final_youngest_desired,
3028 gc_data_global.num_heaps,
3029 gc_data_global.condemned_generation,
3030 gc_data_global.gen0_reduction_count,
3031 gc_data_global.reason,
3032 gc_data_global.global_mechanims_p,
3033 gc_data_global.pause_mode,
3034 gc_data_global.mem_pressure);
3036 #ifdef MULTIPLE_HEAPS
3037 for (int i = 0; i < gc_heap::n_heaps; i++)
3039 gc_heap* hp = gc_heap::g_heaps[i];
3040 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
3041 fire_per_heap_hist_event (current_gc_data_per_heap, hp->heap_number);
3044 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
3045 fire_per_heap_hist_event (current_gc_data_per_heap, heap_number);
3051 gc_heap::dt_low_ephemeral_space_p (gc_tuning_point tp)
3057 case tuning_deciding_condemned_gen:
3058 case tuning_deciding_compaction:
3059 case tuning_deciding_expansion:
3060 case tuning_deciding_full_gc:
3062 ret = (!ephemeral_gen_fit_p (tp));
3065 case tuning_deciding_promote_ephemeral:
3067 size_t new_gen0size = approximate_new_allocation();
3068 ptrdiff_t plan_ephemeral_size = total_ephemeral_size;
3070 dprintf (GTC_LOG, ("h%d: plan eph size is %Id, new gen0 is %Id",
3071 heap_number, plan_ephemeral_size, new_gen0size));
3073 // If we were in no_gc_region we could have allocated a larger than normal segment,
3074 // and the next seg we allocate will be a normal sized seg so if we can't fit the new
3075 // ephemeral generations there, do an ephemeral promotion.
3076 ret = ((soh_segment_size - segment_info_size) < (plan_ephemeral_size + new_gen0size));
3088 gc_heap::dt_high_frag_p (gc_tuning_point tp,
3096 case tuning_deciding_condemned_gen:
3098 dynamic_data* dd = dynamic_data_of (gen_number);
3099 float fragmentation_burden = 0;
3103 ret = (dd_fragmentation (dynamic_data_of (max_generation)) >= dd_max_size(dd));
3104 dprintf (GTC_LOG, ("h%d: frag is %Id, max size is %Id",
3105 heap_number, dd_fragmentation (dd), dd_max_size(dd)));
3109 #ifndef MULTIPLE_HEAPS
3110 if (gen_number == max_generation)
3112 float frag_ratio = (float)(dd_fragmentation (dynamic_data_of (max_generation))) / (float)generation_size (max_generation);
3113 if (frag_ratio > 0.65)
3115 dprintf (GTC_LOG, ("g2 FR: %d%%", (int)(frag_ratio*100)));
3119 #endif //!MULTIPLE_HEAPS
3120 size_t fr = generation_unusable_fragmentation (generation_of (gen_number));
3121 ret = (fr > dd_fragmentation_limit(dd));
3124 fragmentation_burden = (float)fr / generation_size (gen_number);
3125 ret = (fragmentation_burden > dd_v_fragmentation_burden_limit (dd));
3127 dprintf (GTC_LOG, ("h%d: gen%d, frag is %Id, alloc effi: %d%%, unusable frag is %Id, ratio is %d",
3128 heap_number, gen_number, dd_fragmentation (dd),
3129 (int)(100*generation_allocator_efficiency (generation_of (gen_number))),
3130 fr, (int)(fragmentation_burden*100)));
3142 gc_heap::dt_estimate_reclaim_space_p (gc_tuning_point tp, int gen_number)
3148 case tuning_deciding_condemned_gen:
3150 if (gen_number == max_generation)
3152 dynamic_data* dd = dynamic_data_of (gen_number);
3153 size_t maxgen_allocated = (dd_desired_allocation (dd) - dd_new_allocation (dd));
3154 size_t maxgen_total_size = maxgen_allocated + dd_current_size (dd);
3155 size_t est_maxgen_surv = (size_t)((float) (maxgen_total_size) * dd_surv (dd));
3156 size_t est_maxgen_free = maxgen_total_size - est_maxgen_surv + dd_fragmentation (dd);
3158 dprintf (GTC_LOG, ("h%d: Total gen2 size: %Id, est gen2 dead space: %Id (s: %d, allocated: %Id), frag: %Id",
3162 (int)(dd_surv (dd) * 100),
3164 dd_fragmentation (dd)));
3166 uint32_t num_heaps = 1;
3168 #ifdef MULTIPLE_HEAPS
3169 num_heaps = gc_heap::n_heaps;
3170 #endif //MULTIPLE_HEAPS
3172 size_t min_frag_th = min_reclaim_fragmentation_threshold (num_heaps);
3173 dprintf (GTC_LOG, ("h%d, min frag is %Id", heap_number, min_frag_th));
3174 ret = (est_maxgen_free >= min_frag_th);
3190 // DTREVIEW: Right now we only estimate gen2 fragmentation.
3191 // on 64-bit though we should consider gen1 or even gen0 fragmentatioin as
3194 gc_heap::dt_estimate_high_frag_p (gc_tuning_point tp, int gen_number, uint64_t available_mem)
3200 case tuning_deciding_condemned_gen:
3202 if (gen_number == max_generation)
3204 dynamic_data* dd = dynamic_data_of (gen_number);
3205 float est_frag_ratio = 0;
3206 if (dd_current_size (dd) == 0)
3210 else if ((dd_fragmentation (dd) == 0) || (dd_fragmentation (dd) + dd_current_size (dd) == 0))
3216 est_frag_ratio = (float)dd_fragmentation (dd) / (float)(dd_fragmentation (dd) + dd_current_size (dd));
3219 size_t est_frag = (dd_fragmentation (dd) + (size_t)((dd_desired_allocation (dd) - dd_new_allocation (dd)) * est_frag_ratio));
3220 dprintf (GTC_LOG, ("h%d: gen%d: current_size is %Id, frag is %Id, est_frag_ratio is %d%%, estimated frag is %Id",
3223 dd_current_size (dd),
3224 dd_fragmentation (dd),
3225 (int)(est_frag_ratio*100),
3228 uint32_t num_heaps = 1;
3230 #ifdef MULTIPLE_HEAPS
3231 num_heaps = gc_heap::n_heaps;
3232 #endif //MULTIPLE_HEAPS
3233 uint64_t min_frag_th = min_high_fragmentation_threshold(available_mem, num_heaps);
3234 //dprintf (GTC_LOG, ("h%d, min frag is %I64d", heap_number, min_frag_th));
3235 ret = (est_frag >= min_frag_th);
3252 gc_heap::dt_low_card_table_efficiency_p (gc_tuning_point tp)
3258 case tuning_deciding_condemned_gen:
3260 /* promote into max-generation if the card table has too many
3261 * generation faults besides the n -> 0
3263 ret = (generation_skip_ratio < 30);
3275 in_range_for_segment(uint8_t* add, heap_segment* seg)
3277 return ((add >= heap_segment_mem (seg)) && (add < heap_segment_reserved (seg)));
3280 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
3281 // The array we allocate is organized as follows:
3282 // 0th element is the address of the last array we allocated.
3283 // starting from the 1st element are the segment addresses, that's
3284 // what buckets() returns.
3297 bk* buckets() { return (slots + 1); }
3298 uint8_t*& last_slot (bk* arr) { return arr[0].add; }
3301 static sorted_table* make_sorted_table ();
3302 BOOL insert (uint8_t* add, size_t val);;
3303 size_t lookup (uint8_t*& add);
3304 void remove (uint8_t* add);
3306 void delete_sorted_table();
3307 void delete_old_slots();
3308 void enqueue_old_slot(bk* sl);
3309 BOOL ensure_space_for_insert();
3313 sorted_table::make_sorted_table ()
3317 // allocate one more bk to store the older slot address.
3318 sorted_table* res = (sorted_table*)new char [sizeof (sorted_table) + (size + 1) * sizeof (bk)];
3322 res->slots = (bk*)(res + 1);
3329 sorted_table::delete_sorted_table()
3331 if (slots != (bk*)(this+1))
3339 sorted_table::delete_old_slots()
3341 uint8_t* sl = (uint8_t*)old_slots;
3345 sl = last_slot ((bk*)sl);
3351 sorted_table::enqueue_old_slot(bk* sl)
3353 last_slot (sl) = (uint8_t*)old_slots;
3359 sorted_table::lookup (uint8_t*& add)
3361 ptrdiff_t high = (count-1);
3365 bk* buck = buckets();
3368 mid = ((low + high)/2);
3370 if (buck[ti].add > add)
3372 if ((ti > 0) && (buck[ti-1].add <= add))
3374 add = buck[ti-1].add;
3375 return buck[ti - 1].val;
3381 if (buck[ti+1].add > add)
3384 return buck[ti].val;
3394 sorted_table::ensure_space_for_insert()
3398 size = (size * 3)/2;
3399 assert((size * sizeof (bk)) > 0);
3400 bk* res = (bk*)new (nothrow) char [(size + 1) * sizeof (bk)];
3405 last_slot (res) = 0;
3406 memcpy (((bk*)res + 1), buckets(), count * sizeof (bk));
3407 bk* last_old_slots = slots;
3409 if (last_old_slots != (bk*)(this + 1))
3410 enqueue_old_slot (last_old_slots);
3416 sorted_table::insert (uint8_t* add, size_t val)
3418 //grow if no more room
3419 assert (count < size);
3422 ptrdiff_t high = (count-1);
3426 bk* buck = buckets();
3429 mid = ((low + high)/2);
3431 if (buck[ti].add > add)
3433 if ((ti == 0) || (buck[ti-1].add <= add))
3435 // found insertion point
3436 for (ptrdiff_t k = count; k > ti;k--)
3438 buck [k] = buck [k-1];
3449 if (buck[ti+1].add > add)
3451 //found the insertion point
3452 for (ptrdiff_t k = count; k > ti+1;k--)
3454 buck [k] = buck [k-1];
3456 buck[ti+1].add = add;
3457 buck[ti+1].val = val;
3469 sorted_table::remove (uint8_t* add)
3471 ptrdiff_t high = (count-1);
3475 bk* buck = buckets();
3478 mid = ((low + high)/2);
3480 if (buck[ti].add > add)
3482 if (buck[ti-1].add <= add)
3484 // found the guy to remove
3485 for (ptrdiff_t k = ti; k < count; k++)
3486 buck[k-1] = buck[k];
3494 if (buck[ti+1].add > add)
3496 // found the guy to remove
3497 for (ptrdiff_t k = ti+1; k < count; k++)
3498 buck[k-1] = buck[k];
3509 sorted_table::clear()
3512 buckets()[0].add = MAX_PTR;
3514 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
3516 #ifdef SEG_MAPPING_TABLE
3517 #ifdef GROWABLE_SEG_MAPPING_TABLE
3519 uint8_t* align_on_segment (uint8_t* add)
3521 return (uint8_t*)((size_t)(add + (gc_heap::min_segment_size - 1)) & ~(gc_heap::min_segment_size - 1));
3525 uint8_t* align_lower_segment (uint8_t* add)
3527 return (uint8_t*)((size_t)(add) & ~(gc_heap::min_segment_size - 1));
3530 size_t size_seg_mapping_table_of (uint8_t* from, uint8_t* end)
3532 from = align_lower_segment (from);
3533 end = align_on_segment (end);
3534 dprintf (1, ("from: %Ix, end: %Ix, size: %Ix", from, end, sizeof (seg_mapping)*(((size_t)(end - from) >> gc_heap::min_segment_size_shr))));
3535 return sizeof (seg_mapping)*((size_t)(end - from) >> gc_heap::min_segment_size_shr);
3538 // for seg_mapping_table we want it to start from a pointer sized address.
3540 size_t align_for_seg_mapping_table (size_t size)
3542 return ((size + (sizeof (uint8_t*) - 1)) &~ (sizeof (uint8_t*) - 1));
3546 size_t seg_mapping_word_of (uint8_t* add)
3548 return (size_t)add >> gc_heap::min_segment_size_shr;
3550 #else //GROWABLE_SEG_MAPPING_TABLE
3551 BOOL seg_mapping_table_init()
3554 uint64_t total_address_space = (uint64_t)8*1024*1024*1024*1024;
3556 uint64_t total_address_space = (uint64_t)4*1024*1024*1024;
3559 size_t num_entries = (size_t)(total_address_space >> gc_heap::min_segment_size_shr);
3560 seg_mapping_table = new seg_mapping[num_entries];
3562 if (seg_mapping_table)
3564 memset (seg_mapping_table, 0, num_entries * sizeof (seg_mapping));
3565 dprintf (1, ("created %d entries for heap mapping (%Id bytes)",
3566 num_entries, (num_entries * sizeof (seg_mapping))));
3571 dprintf (1, ("failed to create %d entries for heap mapping (%Id bytes)",
3572 num_entries, (num_entries * sizeof (seg_mapping))));
3576 #endif //GROWABLE_SEG_MAPPING_TABLE
3578 #ifdef FEATURE_BASICFREEZE
3580 size_t ro_seg_begin_index (heap_segment* seg)
3582 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3583 begin_index = max (begin_index, (size_t)g_gc_lowest_address >> gc_heap::min_segment_size_shr);
3588 size_t ro_seg_end_index (heap_segment* seg)
3590 size_t end_index = (size_t)(heap_segment_reserved (seg) - 1) >> gc_heap::min_segment_size_shr;
3591 end_index = min (end_index, (size_t)g_gc_highest_address >> gc_heap::min_segment_size_shr);
3595 void seg_mapping_table_add_ro_segment (heap_segment* seg)
3597 #ifdef GROWABLE_SEG_MAPPING_TABLE
3598 if ((heap_segment_reserved (seg) <= g_gc_lowest_address) || (heap_segment_mem (seg) >= g_gc_highest_address))
3600 #endif //GROWABLE_SEG_MAPPING_TABLE
3602 for (size_t entry_index = ro_seg_begin_index (seg); entry_index <= ro_seg_end_index (seg); entry_index++)
3603 seg_mapping_table[entry_index].seg1 = (heap_segment*)((size_t)seg_mapping_table[entry_index].seg1 | ro_in_entry);
3606 void seg_mapping_table_remove_ro_segment (heap_segment* seg)
3608 UNREFERENCED_PARAMETER(seg);
3610 // POSSIBLE PERF TODO: right now we are not doing anything because we can't simply remove the flag. If it proves
3611 // to be a perf problem, we can search in the current ro segs and see if any lands in this range and only
3612 // remove the flag if none lands in this range.
3616 heap_segment* ro_segment_lookup (uint8_t* o)
3618 uint8_t* ro_seg_start = o;
3619 heap_segment* seg = (heap_segment*)gc_heap::seg_table->lookup (ro_seg_start);
3621 if (ro_seg_start && in_range_for_segment (o, seg))
3627 #endif //FEATURE_BASICFREEZE
3629 void gc_heap::seg_mapping_table_add_segment (heap_segment* seg, gc_heap* hp)
3631 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3632 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3633 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3634 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3635 seg_mapping* end_entry = &seg_mapping_table[end_index];
3637 dprintf (1, ("adding seg %Ix(%d)-%Ix(%d)",
3638 seg, begin_index, heap_segment_reserved (seg), end_index));
3640 dprintf (1, ("before add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3641 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3642 end_index, (seg_mapping_table[end_index].boundary + 1)));
3644 #ifdef MULTIPLE_HEAPS
3645 #ifdef SIMPLE_DPRINTF
3646 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end %d: h0: %Ix(%d), h1: %Ix(%d)",
3647 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3648 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3649 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3650 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3651 #endif //SIMPLE_DPRINTF
3652 assert (end_entry->boundary == 0);
3653 assert (end_entry->h0 == 0);
3655 assert (begin_entry->h1 == 0);
3656 begin_entry->h1 = hp;
3658 UNREFERENCED_PARAMETER(hp);
3659 #endif //MULTIPLE_HEAPS
3661 end_entry->boundary = (uint8_t*)seg_end;
3663 dprintf (1, ("set entry %d seg1 and %d seg0 to %Ix", begin_index, end_index, seg));
3664 assert ((begin_entry->seg1 == 0) || ((size_t)(begin_entry->seg1) == ro_in_entry));
3665 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) | (size_t)seg);
3666 end_entry->seg0 = seg;
3668 // for every entry inbetween we need to set its heap too.
3669 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3671 assert (seg_mapping_table[entry_index].boundary == 0);
3672 #ifdef MULTIPLE_HEAPS
3673 assert (seg_mapping_table[entry_index].h0 == 0);
3674 seg_mapping_table[entry_index].h1 = hp;
3675 #endif //MULTIPLE_HEAPS
3676 seg_mapping_table[entry_index].seg1 = seg;
3679 dprintf (1, ("after add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3680 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3681 end_index, (seg_mapping_table[end_index].boundary + 1)));
3682 #if defined(MULTIPLE_HEAPS) && defined(SIMPLE_DPRINTF)
3683 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end: %d h0: %Ix(%d), h1: %Ix(%d)",
3684 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3685 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3686 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3687 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3688 #endif //MULTIPLE_HEAPS && SIMPLE_DPRINTF
3691 void gc_heap::seg_mapping_table_remove_segment (heap_segment* seg)
3693 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3694 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3695 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3696 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3697 seg_mapping* end_entry = &seg_mapping_table[end_index];
3698 dprintf (1, ("removing seg %Ix(%d)-%Ix(%d)",
3699 seg, begin_index, heap_segment_reserved (seg), end_index));
3701 assert (end_entry->boundary == (uint8_t*)seg_end);
3702 end_entry->boundary = 0;
3704 #ifdef MULTIPLE_HEAPS
3705 gc_heap* hp = heap_segment_heap (seg);
3706 assert (end_entry->h0 == hp);
3708 assert (begin_entry->h1 == hp);
3709 begin_entry->h1 = 0;
3710 #endif //MULTIPLE_HEAPS
3712 assert (begin_entry->seg1 != 0);
3713 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) & ro_in_entry);
3714 end_entry->seg0 = 0;
3716 // for every entry inbetween we need to reset its heap too.
3717 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3719 assert (seg_mapping_table[entry_index].boundary == 0);
3720 #ifdef MULTIPLE_HEAPS
3721 assert (seg_mapping_table[entry_index].h0 == 0);
3722 assert (seg_mapping_table[entry_index].h1 == hp);
3723 seg_mapping_table[entry_index].h1 = 0;
3724 #endif //MULTIPLE_HEAPS
3725 seg_mapping_table[entry_index].seg1 = 0;
3728 dprintf (1, ("after remove: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3729 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3730 end_index, (seg_mapping_table[end_index].boundary + 1)));
3731 #ifdef MULTIPLE_HEAPS
3732 dprintf (1, ("begin %d: h0: %Ix, h1: %Ix; end: %d h0: %Ix, h1: %Ix",
3733 begin_index, (uint8_t*)(begin_entry->h0), (uint8_t*)(begin_entry->h1),
3734 end_index, (uint8_t*)(end_entry->h0), (uint8_t*)(end_entry->h1)));
3735 #endif //MULTIPLE_HEAPS
3738 #ifdef MULTIPLE_HEAPS
3740 gc_heap* seg_mapping_table_heap_of_worker (uint8_t* o)
3742 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3743 seg_mapping* entry = &seg_mapping_table[index];
3745 gc_heap* hp = ((o > entry->boundary) ? entry->h1 : entry->h0);
3747 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, h0: %Ix, seg0: %Ix, h1: %Ix, seg1: %Ix",
3748 o, index, (entry->boundary + 1),
3749 (uint8_t*)(entry->h0), (uint8_t*)(entry->seg0),
3750 (uint8_t*)(entry->h1), (uint8_t*)(entry->seg1)));
3753 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3754 #ifdef FEATURE_BASICFREEZE
3755 if ((size_t)seg & ro_in_entry)
3756 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3757 #endif //FEATURE_BASICFREEZE
3761 if (in_range_for_segment (o, seg))
3763 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, seg, (uint8_t*)heap_segment_allocated (seg)));
3767 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg",
3768 seg, (uint8_t*)heap_segment_allocated (seg), o));
3773 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3780 gc_heap* seg_mapping_table_heap_of (uint8_t* o)
3782 #ifdef GROWABLE_SEG_MAPPING_TABLE
3783 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3785 #endif //GROWABLE_SEG_MAPPING_TABLE
3787 return seg_mapping_table_heap_of_worker (o);
3790 gc_heap* seg_mapping_table_heap_of_gc (uint8_t* o)
3792 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3793 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3795 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3797 return seg_mapping_table_heap_of_worker (o);
3799 #endif //MULTIPLE_HEAPS
3801 // Only returns a valid seg if we can actually find o on the seg.
3802 heap_segment* seg_mapping_table_segment_of (uint8_t* o)
3804 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3805 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3806 #ifdef FEATURE_BASICFREEZE
3807 return ro_segment_lookup (o);
3810 #endif //FEATURE_BASICFREEZE
3811 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3813 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3814 seg_mapping* entry = &seg_mapping_table[index];
3816 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, seg0: %Ix, seg1: %Ix",
3817 o, index, (entry->boundary + 1),
3818 (uint8_t*)(entry->seg0), (uint8_t*)(entry->seg1)));
3820 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3821 #ifdef FEATURE_BASICFREEZE
3822 if ((size_t)seg & ro_in_entry)
3823 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3824 #endif //FEATURE_BASICFREEZE
3828 if (in_range_for_segment (o, seg))
3830 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg)));
3834 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg, setting it to 0",
3835 (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg), o));
3841 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3844 #ifdef FEATURE_BASICFREEZE
3845 // TODO: This was originally written assuming that the seg_mapping_table would always contain entries for ro
3846 // segments whenever the ro segment falls into the [g_gc_lowest_address,g_gc_highest_address) range. I.e., it had an
3847 // extra "&& (size_t)(entry->seg1) & ro_in_entry" expression. However, at the moment, grow_brick_card_table does
3848 // not correctly go through the ro segments and add them back to the seg_mapping_table when the [lowest,highest)
3849 // range changes. We should probably go ahead and modify grow_brick_card_table and put back the
3850 // "&& (size_t)(entry->seg1) & ro_in_entry" here.
3853 seg = ro_segment_lookup (o);
3854 if (seg && !in_range_for_segment (o, seg))
3857 #endif //FEATURE_BASICFREEZE
3861 #endif //SEG_MAPPING_TABLE
3863 size_t gcard_of ( uint8_t*);
3865 #define memref(i) *(uint8_t**)(i)
3868 #define GC_MARKED (size_t)0x1
3869 #define slot(i, j) ((uint8_t**)(i))[j+1]
3871 #define free_object_base_size (plug_skew + sizeof(ArrayBase))
3873 class CObjectHeader : public Object
3877 #if defined(FEATURE_REDHAWK) || defined(BUILD_AS_STANDALONE)
3878 // The GC expects the following methods that are provided by the Object class in the CLR but not provided
3879 // by Redhawk's version of Object.
3880 uint32_t GetNumComponents()
3882 return ((ArrayBase *)this)->GetNumComponents();
3885 void Validate(BOOL bDeep=TRUE, BOOL bVerifyNextHeader = TRUE)
3887 UNREFERENCED_PARAMETER(bVerifyNextHeader);
3892 MethodTable * pMT = GetMethodTable();
3894 _ASSERTE(pMT->SanityCheck());
3896 bool noRangeChecks =
3897 (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_RANGE_CHECKS) == GCConfig::HEAPVERIFY_NO_RANGE_CHECKS;
3899 BOOL fSmallObjectHeapPtr = FALSE, fLargeObjectHeapPtr = FALSE;
3902 fSmallObjectHeapPtr = g_theGCHeap->IsHeapPointer(this, TRUE);
3903 if (!fSmallObjectHeapPtr)
3904 fLargeObjectHeapPtr = g_theGCHeap->IsHeapPointer(this);
3906 _ASSERTE(fSmallObjectHeapPtr || fLargeObjectHeapPtr);
3909 #ifdef FEATURE_STRUCTALIGN
3910 _ASSERTE(IsStructAligned((uint8_t *)this, GetMethodTable()->GetBaseAlignment()));
3911 #endif // FEATURE_STRUCTALIGN
3913 #ifdef FEATURE_64BIT_ALIGNMENT
3914 if (pMT->RequiresAlign8())
3916 _ASSERTE((((size_t)this) & 0x7) == (pMT->IsValueType() ? 4U : 0U));
3918 #endif // FEATURE_64BIT_ALIGNMENT
3921 if (bDeep && (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC))
3922 g_theGCHeap->ValidateObjectMember(this);
3924 if (fSmallObjectHeapPtr)
3926 #ifdef FEATURE_BASICFREEZE
3927 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT) || g_theGCHeap->IsInFrozenSegment(this));
3929 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT));
3934 void ValidatePromote(ScanContext *sc, uint32_t flags)
3936 UNREFERENCED_PARAMETER(sc);
3937 UNREFERENCED_PARAMETER(flags);
3942 void ValidateHeap(Object *from, BOOL bDeep)
3944 UNREFERENCED_PARAMETER(from);
3946 Validate(bDeep, FALSE);
3949 ADIndex GetAppDomainIndex()
3951 return (ADIndex)RH_DEFAULT_DOMAIN_ID;
3953 #endif //FEATURE_REDHAWK
3957 // Header Status Information
3960 MethodTable *GetMethodTable() const
3962 return( (MethodTable *) (((size_t) RawGetMethodTable()) & (~(GC_MARKED))));
3967 RawSetMethodTable((MethodTable *) (((size_t) RawGetMethodTable()) | GC_MARKED));
3970 BOOL IsMarked() const
3972 return !!(((size_t)RawGetMethodTable()) & GC_MARKED);
3977 assert (!(gc_heap::settings.concurrent));
3978 GetHeader()->SetGCBit();
3981 BOOL IsPinned() const
3983 return !!((((CObjectHeader*)this)->GetHeader()->GetBits()) & BIT_SBLK_GC_RESERVE);
3988 RawSetMethodTable( GetMethodTable() );
3991 CGCDesc *GetSlotMap ()
3993 assert (GetMethodTable()->ContainsPointers());
3994 return CGCDesc::GetCGCDescFromMT(GetMethodTable());
3997 void SetFree(size_t size)
3999 assert (size >= free_object_base_size);
4001 assert (g_gc_pFreeObjectMethodTable->GetBaseSize() == free_object_base_size);
4002 assert (g_gc_pFreeObjectMethodTable->RawGetComponentSize() == 1);
4004 RawSetMethodTable( g_gc_pFreeObjectMethodTable );
4006 size_t* numComponentsPtr = (size_t*) &((uint8_t*) this)[ArrayBase::GetOffsetOfNumComponents()];
4007 *numComponentsPtr = size - free_object_base_size;
4009 //This introduces a bug in the free list management.
4010 //((void**) this)[-1] = 0; // clear the sync block,
4011 assert (*numComponentsPtr >= 0);
4012 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
4013 memset (((uint8_t*)this)+sizeof(ArrayBase), 0xcc, *numComponentsPtr);
4014 #endif //VERIFY_HEAP
4019 size_t size = free_object_base_size - plug_skew;
4021 // since we only need to clear 2 ptr size, we do it manually
4022 PTR_PTR m = (PTR_PTR) this;
4023 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
4027 BOOL IsFree () const
4029 return (GetMethodTable() == g_gc_pFreeObjectMethodTable);
4032 #ifdef FEATURE_STRUCTALIGN
4033 int GetRequiredAlignment () const
4035 return GetMethodTable()->GetRequiredAlignment();
4037 #endif // FEATURE_STRUCTALIGN
4039 BOOL ContainsPointers() const
4041 return GetMethodTable()->ContainsPointers();
4044 #ifdef COLLECTIBLE_CLASS
4045 BOOL Collectible() const
4047 return GetMethodTable()->Collectible();
4050 FORCEINLINE BOOL ContainsPointersOrCollectible() const
4052 MethodTable *pMethodTable = GetMethodTable();
4053 return (pMethodTable->ContainsPointers() || pMethodTable->Collectible());
4055 #endif //COLLECTIBLE_CLASS
4057 Object* GetObjectBase() const
4059 return (Object*) this;
4063 #define header(i) ((CObjectHeader*)(i))
4065 #define free_list_slot(x) ((uint8_t**)(x))[2]
4066 #define free_list_undo(x) ((uint8_t**)(x))[-1]
4067 #define UNDO_EMPTY ((uint8_t*)1)
4071 void set_plug_padded (uint8_t* node)
4073 header(node)->SetMarked();
4076 void clear_plug_padded (uint8_t* node)
4078 header(node)->ClearMarked();
4081 BOOL is_plug_padded (uint8_t* node)
4083 return header(node)->IsMarked();
4086 inline void set_plug_padded (uint8_t* node){}
4087 inline void clear_plug_padded (uint8_t* node){}
4089 BOOL is_plug_padded (uint8_t* node){return FALSE;}
4090 #endif //SHORT_PLUGS
4093 inline size_t unused_array_size(uint8_t * p)
4095 assert(((CObjectHeader*)p)->IsFree());
4097 size_t* numComponentsPtr = (size_t*)(p + ArrayBase::GetOffsetOfNumComponents());
4098 return free_object_base_size + *numComponentsPtr;
4101 heap_segment* heap_segment_rw (heap_segment* ns)
4103 if ((ns == 0) || !heap_segment_read_only_p (ns))
4111 ns = heap_segment_next (ns);
4112 } while ((ns != 0) && heap_segment_read_only_p (ns));
4117 //returns the next non ro segment.
4118 heap_segment* heap_segment_next_rw (heap_segment* seg)
4120 heap_segment* ns = heap_segment_next (seg);
4121 return heap_segment_rw (ns);
4124 // returns the segment before seg.
4125 heap_segment* heap_segment_prev_rw (heap_segment* begin, heap_segment* seg)
4127 assert (begin != 0);
4128 heap_segment* prev = begin;
4129 heap_segment* current = heap_segment_next_rw (begin);
4131 while (current && current != seg)
4134 current = heap_segment_next_rw (current);
4147 // returns the segment before seg.
4148 heap_segment* heap_segment_prev (heap_segment* begin, heap_segment* seg)
4150 assert (begin != 0);
4151 heap_segment* prev = begin;
4152 heap_segment* current = heap_segment_next (begin);
4154 while (current && current != seg)
4157 current = heap_segment_next (current);
4170 heap_segment* heap_segment_in_range (heap_segment* ns)
4172 if ((ns == 0) || heap_segment_in_range_p (ns))
4180 ns = heap_segment_next (ns);
4181 } while ((ns != 0) && !heap_segment_in_range_p (ns));
4186 heap_segment* heap_segment_next_in_range (heap_segment* seg)
4188 heap_segment* ns = heap_segment_next (seg);
4189 return heap_segment_in_range (ns);
4194 uint8_t* memory_base;
4199 imemory_data *initial_memory;
4200 imemory_data *initial_normal_heap; // points into initial_memory_array
4201 imemory_data *initial_large_heap; // points into initial_memory_array
4203 size_t block_size_normal;
4204 size_t block_size_large;
4206 size_t block_count; // # of blocks in each
4207 size_t current_block_normal;
4208 size_t current_block_large;
4217 size_t allocation_pattern;
4218 } initial_memory_details;
4220 initial_memory_details memory_details;
4222 BOOL reserve_initial_memory (size_t normal_size, size_t large_size, size_t num_heaps)
4224 BOOL reserve_success = FALSE;
4226 // should only be called once
4227 assert (memory_details.initial_memory == 0);
4229 memory_details.initial_memory = new (nothrow) imemory_data[num_heaps*2];
4230 if (memory_details.initial_memory == 0)
4232 dprintf (2, ("failed to reserve %Id bytes for imemory_data", num_heaps*2*sizeof(imemory_data)));
4236 memory_details.initial_normal_heap = memory_details.initial_memory;
4237 memory_details.initial_large_heap = memory_details.initial_memory + num_heaps;
4238 memory_details.block_size_normal = normal_size;
4239 memory_details.block_size_large = large_size;
4240 memory_details.block_count = num_heaps;
4242 memory_details.current_block_normal = 0;
4243 memory_details.current_block_large = 0;
4245 g_gc_lowest_address = MAX_PTR;
4246 g_gc_highest_address = 0;
4248 if (((size_t)MAX_PTR - large_size) < normal_size)
4250 // we are already overflowing with just one heap.
4251 dprintf (2, ("0x%Ix + 0x%Ix already overflow", normal_size, large_size));
4255 if (((size_t)MAX_PTR / memory_details.block_count) < (normal_size + large_size))
4257 dprintf (2, ("(0x%Ix + 0x%Ix)*0x%Ix overflow", normal_size, large_size, memory_details.block_count));
4261 size_t requestedMemory = memory_details.block_count * (normal_size + large_size);
4263 uint8_t* allatonce_block = (uint8_t*)virtual_alloc (requestedMemory);
4264 if (allatonce_block)
4266 g_gc_lowest_address = allatonce_block;
4267 g_gc_highest_address = allatonce_block + (memory_details.block_count * (large_size + normal_size));
4268 memory_details.allocation_pattern = initial_memory_details::ALLATONCE;
4270 for(size_t i = 0; i < memory_details.block_count; i++)
4272 memory_details.initial_normal_heap[i].memory_base = allatonce_block + (i*normal_size);
4273 memory_details.initial_large_heap[i].memory_base = allatonce_block +
4274 (memory_details.block_count*normal_size) + (i*large_size);
4275 reserve_success = TRUE;
4280 // try to allocate 2 blocks
4283 b1 = (uint8_t*)virtual_alloc (memory_details.block_count * normal_size);
4286 b2 = (uint8_t*)virtual_alloc (memory_details.block_count * large_size);
4289 memory_details.allocation_pattern = initial_memory_details::TWO_STAGE;
4290 g_gc_lowest_address = min(b1,b2);
4291 g_gc_highest_address = max(b1 + memory_details.block_count*normal_size,
4292 b2 + memory_details.block_count*large_size);
4293 for(size_t i = 0; i < memory_details.block_count; i++)
4295 memory_details.initial_normal_heap[i].memory_base = b1 + (i*normal_size);
4296 memory_details.initial_large_heap[i].memory_base = b2 + (i*large_size);
4297 reserve_success = TRUE;
4302 // b2 allocation failed, we'll go on to try allocating each block.
4303 // We could preserve the b1 alloc, but code complexity increases
4304 virtual_free (b1, memory_details.block_count * normal_size);
4308 if ((b2==NULL) && ( memory_details.block_count > 1))
4310 memory_details.allocation_pattern = initial_memory_details::EACH_BLOCK;
4312 imemory_data *current_block = memory_details.initial_memory;
4313 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4315 size_t block_size = ((i < memory_details.block_count) ?
4316 memory_details.block_size_normal :
4317 memory_details.block_size_large);
4318 current_block->memory_base =
4319 (uint8_t*)virtual_alloc (block_size);
4320 if (current_block->memory_base == 0)
4322 // Free the blocks that we've allocated so far
4323 current_block = memory_details.initial_memory;
4324 for(size_t j = 0; j < i; j++, current_block++){
4325 if (current_block->memory_base != 0){
4326 block_size = ((j < memory_details.block_count) ?
4327 memory_details.block_size_normal :
4328 memory_details.block_size_large);
4329 virtual_free (current_block->memory_base , block_size);
4332 reserve_success = FALSE;
4337 if (current_block->memory_base < g_gc_lowest_address)
4338 g_gc_lowest_address = current_block->memory_base;
4339 if (((uint8_t *) current_block->memory_base + block_size) > g_gc_highest_address)
4340 g_gc_highest_address = (current_block->memory_base + block_size);
4342 reserve_success = TRUE;
4347 return reserve_success;
4350 void destroy_initial_memory()
4352 if (memory_details.initial_memory != NULL)
4354 if (memory_details.allocation_pattern == initial_memory_details::ALLATONCE)
4356 virtual_free(memory_details.initial_memory[0].memory_base,
4357 memory_details.block_count*(memory_details.block_size_normal +
4358 memory_details.block_size_large));
4360 else if (memory_details.allocation_pattern == initial_memory_details::TWO_STAGE)
4362 virtual_free (memory_details.initial_normal_heap[0].memory_base,
4363 memory_details.block_count*memory_details.block_size_normal);
4365 virtual_free (memory_details.initial_large_heap[0].memory_base,
4366 memory_details.block_count*memory_details.block_size_large);
4370 assert (memory_details.allocation_pattern == initial_memory_details::EACH_BLOCK);
4371 imemory_data *current_block = memory_details.initial_memory;
4372 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4374 size_t block_size = (i < memory_details.block_count) ? memory_details.block_size_normal :
4375 memory_details.block_size_large;
4376 if (current_block->memory_base != NULL)
4378 virtual_free (current_block->memory_base, block_size);
4383 delete [] memory_details.initial_memory;
4384 memory_details.initial_memory = NULL;
4385 memory_details.initial_normal_heap = NULL;
4386 memory_details.initial_large_heap = NULL;
4390 void* next_initial_memory (size_t size)
4392 assert ((size == memory_details.block_size_normal) || (size == memory_details.block_size_large));
4395 if ((size != memory_details.block_size_normal) ||
4396 ((memory_details.current_block_normal == memory_details.block_count) &&
4397 (memory_details.block_size_normal == memory_details.block_size_large)))
4399 // If the block sizes are the same, flow block requests from normal to large
4400 assert (memory_details.current_block_large < memory_details.block_count);
4401 assert (memory_details.initial_large_heap != 0);
4403 res = memory_details.initial_large_heap[memory_details.current_block_large].memory_base;
4404 memory_details.current_block_large++;
4408 assert (memory_details.current_block_normal < memory_details.block_count);
4409 assert (memory_details.initial_normal_heap != NULL);
4411 res = memory_details.initial_normal_heap[memory_details.current_block_normal].memory_base;
4412 memory_details.current_block_normal++;
4418 heap_segment* get_initial_segment (size_t size, int h_number)
4420 void* mem = next_initial_memory (size);
4421 heap_segment* res = gc_heap::make_heap_segment ((uint8_t*)mem, size , h_number);
4426 void* virtual_alloc (size_t size)
4428 size_t requested_size = size;
4430 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4432 gc_heap::reserved_memory_limit =
4433 GCScan::AskForMoreReservedMemory (gc_heap::reserved_memory_limit, requested_size);
4434 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4440 uint32_t flags = VirtualReserveFlags::None;
4441 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4442 if (virtual_alloc_hardware_write_watch)
4444 flags = VirtualReserveFlags::WriteWatch;
4446 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4447 void* prgmem = GCToOSInterface::VirtualReserve (requested_size, card_size * card_word_width, flags);
4448 void *aligned_mem = prgmem;
4450 // We don't want (prgmem + size) to be right at the end of the address space
4451 // because we'd have to worry about that everytime we do (address + size).
4452 // We also want to make sure that we leave LARGE_OBJECT_SIZE at the end
4453 // so we allocate a small object we don't need to worry about overflow there
4454 // when we do alloc_ptr+size.
4457 uint8_t* end_mem = (uint8_t*)prgmem + requested_size;
4459 if ((end_mem == 0) || ((size_t)(MAX_PTR - end_mem) <= END_SPACE_AFTER_GC))
4461 GCToOSInterface::VirtualRelease (prgmem, requested_size);
4462 dprintf (2, ("Virtual Alloc size %Id returned memory right against 4GB [%Ix, %Ix[ - discarding",
4463 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4471 gc_heap::reserved_memory += requested_size;
4474 dprintf (2, ("Virtual Alloc size %Id: [%Ix, %Ix[",
4475 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4480 void virtual_free (void* add, size_t size)
4482 GCToOSInterface::VirtualRelease (add, size);
4483 gc_heap::reserved_memory -= size;
4484 dprintf (2, ("Virtual Free size %Id: [%Ix, %Ix[",
4485 size, (size_t)add, (size_t)((uint8_t*)add+size)));
4488 static size_t get_valid_segment_size (BOOL large_seg=FALSE)
4490 size_t seg_size, initial_seg_size;
4494 initial_seg_size = INITIAL_ALLOC;
4495 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize());
4499 initial_seg_size = LHEAP_ALLOC;
4500 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize()) / 2;
4503 #ifdef MULTIPLE_HEAPS
4508 if (g_num_processors > 4)
4509 initial_seg_size /= 2;
4510 if (g_num_processors > 8)
4511 initial_seg_size /= 2;
4513 #endif //MULTIPLE_HEAPS
4515 // if seg_size is small but not 0 (0 is default if config not set)
4516 // then set the segment to the minimum size
4517 if (!g_theGCHeap->IsValidSegmentSize(seg_size))
4519 // if requested size is between 1 byte and 4MB, use min
4520 if ((seg_size >> 1) && !(seg_size >> 22))
4521 seg_size = 1024*1024*4;
4523 seg_size = initial_seg_size;
4526 #ifdef SEG_MAPPING_TABLE
4528 seg_size = round_up_power2 (seg_size);
4530 seg_size = round_down_power2 (seg_size);
4532 #endif //SEG_MAPPING_TABLE
4538 gc_heap::compute_new_ephemeral_size()
4540 int eph_gen_max = max_generation - 1 - (settings.promotion ? 1 : 0);
4541 size_t padding_size = 0;
4543 for (int i = 0; i <= eph_gen_max; i++)
4545 dynamic_data* dd = dynamic_data_of (i);
4546 total_ephemeral_size += (dd_survived_size (dd) - dd_pinned_survived_size (dd));
4547 #ifdef RESPECT_LARGE_ALIGNMENT
4548 total_ephemeral_size += dd_num_npinned_plugs (dd) * switch_alignment_size (FALSE);
4549 #endif //RESPECT_LARGE_ALIGNMENT
4550 #ifdef FEATURE_STRUCTALIGN
4551 total_ephemeral_size += dd_num_npinned_plugs (dd) * MAX_STRUCTALIGN;
4552 #endif //FEATURE_STRUCTALIGN
4555 padding_size += dd_padding_size (dd);
4556 #endif //SHORT_PLUGS
4559 total_ephemeral_size += eph_gen_starts_size;
4561 #ifdef RESPECT_LARGE_ALIGNMENT
4562 size_t planned_ephemeral_size = heap_segment_plan_allocated (ephemeral_heap_segment) -
4563 generation_plan_allocation_start (generation_of (max_generation-1));
4564 total_ephemeral_size = min (total_ephemeral_size, planned_ephemeral_size);
4565 #endif //RESPECT_LARGE_ALIGNMENT
4568 total_ephemeral_size = Align ((size_t)((double)total_ephemeral_size * short_plugs_pad_ratio) + 1);
4569 total_ephemeral_size += Align (DESIRED_PLUG_LENGTH);
4570 #endif //SHORT_PLUGS
4572 dprintf (3, ("total ephemeral size is %Ix, padding %Ix(%Ix)",
4573 total_ephemeral_size,
4574 padding_size, (total_ephemeral_size - padding_size)));
4578 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
4582 gc_heap::soh_get_segment_to_expand()
4584 size_t size = soh_segment_size;
4586 ordered_plug_indices_init = FALSE;
4587 use_bestfit = FALSE;
4589 //compute the size of the new ephemeral heap segment.
4590 compute_new_ephemeral_size();
4592 if ((settings.pause_mode != pause_low_latency) &&
4593 (settings.pause_mode != pause_no_gc)
4594 #ifdef BACKGROUND_GC
4595 && (!recursive_gc_sync::background_running_p())
4596 #endif //BACKGROUND_GC
4599 allocator* gen_alloc = ((settings.condemned_generation == max_generation) ? 0 :
4600 generation_allocator (generation_of (max_generation)));
4601 dprintf (2, ("(gen%d)soh_get_segment_to_expand", settings.condemned_generation));
4603 // try to find one in the gen 2 segment list, search backwards because the first segments
4604 // tend to be more compact than the later ones.
4605 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
4607 PREFIX_ASSUME(fseg != NULL);
4609 #ifdef SEG_REUSE_STATS
4611 #endif //SEG_REUSE_STATS
4613 heap_segment* seg = ephemeral_heap_segment;
4614 while ((seg = heap_segment_prev_rw (fseg, seg)) && (seg != fseg))
4616 #ifdef SEG_REUSE_STATS
4618 #endif //SEG_REUSE_STATS
4620 if (can_expand_into_p (seg, size/3, total_ephemeral_size, gen_alloc))
4622 get_gc_data_per_heap()->set_mechanism (gc_heap_expand,
4623 (use_bestfit ? expand_reuse_bestfit : expand_reuse_normal));
4624 if (settings.condemned_generation == max_generation)
4628 build_ordered_free_spaces (seg);
4629 dprintf (GTC_LOG, ("can use best fit"));
4632 #ifdef SEG_REUSE_STATS
4633 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse",
4634 settings.condemned_generation, try_reuse));
4635 #endif //SEG_REUSE_STATS
4636 dprintf (GTC_LOG, ("max_gen: Found existing segment to expand into %Ix", (size_t)seg));
4641 #ifdef SEG_REUSE_STATS
4642 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse - returning",
4643 settings.condemned_generation, try_reuse));
4644 #endif //SEG_REUSE_STATS
4645 dprintf (GTC_LOG, ("max_gen-1: Found existing segment to expand into %Ix", (size_t)seg));
4647 // If we return 0 here, the allocator will think since we are short on end
4648 // of seg we neeed to trigger a full compacting GC. So if sustained low latency
4649 // is set we should acquire a new seg instead, that way we wouldn't be short.
4650 // The real solution, of course, is to actually implement seg reuse in gen1.
4651 if (settings.pause_mode != pause_sustained_low_latency)
4653 dprintf (GTC_LOG, ("max_gen-1: SustainedLowLatency is set, acquire a new seg"));
4654 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_next_full_gc);
4662 heap_segment* result = get_segment (size, FALSE);
4666 #ifdef BACKGROUND_GC
4667 if (current_c_gc_state == c_gc_state_planning)
4669 // When we expand heap during bgc sweep, we set the seg to be swept so
4670 // we'll always look at cards for objects on the new segment.
4671 result->flags |= heap_segment_flags_swept;
4673 #endif //BACKGROUND_GC
4675 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(result),
4676 (size_t)(heap_segment_reserved (result) - heap_segment_mem(result)),
4677 gc_etw_segment_small_object_heap);
4680 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, (result ? expand_new_seg : expand_no_memory));
4684 dprintf (2, ("h%d: failed to allocate a new segment!", heap_number));
4688 #ifdef MULTIPLE_HEAPS
4689 heap_segment_heap (result) = this;
4690 #endif //MULTIPLE_HEAPS
4693 dprintf (GTC_LOG, ("(gen%d)creating new segment %Ix", settings.condemned_generation, result));
4698 #pragma warning(default:4706)
4701 //returns 0 in case of allocation failure
4703 gc_heap::get_segment (size_t size, BOOL loh_p)
4705 heap_segment* result = 0;
4707 if (segment_standby_list != 0)
4709 result = segment_standby_list;
4710 heap_segment* last = 0;
4713 size_t hs = (size_t)(heap_segment_reserved (result) - (uint8_t*)result);
4714 if ((hs >= size) && ((hs / 2) < size))
4716 dprintf (2, ("Hoarded segment %Ix found", (size_t) result));
4719 heap_segment_next (last) = heap_segment_next (result);
4723 segment_standby_list = heap_segment_next (result);
4730 result = heap_segment_next (result);
4737 init_heap_segment (result);
4738 #ifdef BACKGROUND_GC
4739 if (should_commit_mark_array())
4741 dprintf (GC_TABLE_LOG, ("hoarded seg %Ix, mark_array is %Ix", result, mark_array));
4742 if (!commit_mark_array_new_seg (__this, result))
4744 dprintf (GC_TABLE_LOG, ("failed to commit mark array for hoarded seg"));
4745 // If we can't use it we need to thread it back.
4746 if (segment_standby_list != 0)
4748 heap_segment_next (result) = segment_standby_list;
4749 segment_standby_list = result;
4753 segment_standby_list = result;
4759 #endif //BACKGROUND_GC
4761 #ifdef SEG_MAPPING_TABLE
4763 seg_mapping_table_add_segment (result, __this);
4764 #endif //SEG_MAPPING_TABLE
4769 #ifndef SEG_MAPPING_TABLE
4770 if (!seg_table->ensure_space_for_insert ())
4772 #endif //SEG_MAPPING_TABLE
4773 void* mem = virtual_alloc (size);
4776 fgm_result.set_fgm (fgm_reserve_segment, size, loh_p);
4780 result = gc_heap::make_heap_segment ((uint8_t*)mem, size, heap_number);
4786 if (mem < g_gc_lowest_address)
4788 start = (uint8_t*)mem;
4792 start = (uint8_t*)g_gc_lowest_address;
4795 if (((uint8_t*)mem + size) > g_gc_highest_address)
4797 end = (uint8_t*)mem + size;
4801 end = (uint8_t*)g_gc_highest_address;
4804 if (gc_heap::grow_brick_card_tables (start, end, size, result, __this, loh_p) != 0)
4806 virtual_free (mem, size);
4812 fgm_result.set_fgm (fgm_commit_segment_beg, SEGMENT_INITIAL_COMMIT, loh_p);
4813 virtual_free (mem, size);
4818 #ifdef SEG_MAPPING_TABLE
4819 seg_mapping_table_add_segment (result, __this);
4820 #else //SEG_MAPPING_TABLE
4821 gc_heap::seg_table->insert ((uint8_t*)result, delta);
4822 #endif //SEG_MAPPING_TABLE
4826 #ifdef BACKGROUND_GC
4829 ::record_changed_seg ((uint8_t*)result, heap_segment_reserved (result),
4830 settings.gc_index, current_bgc_state,
4832 bgc_verify_mark_array_cleared (result);
4834 #endif //BACKGROUND_GC
4836 dprintf (GC_TABLE_LOG, ("h%d: new seg: %Ix-%Ix (%Id)", heap_number, result, ((uint8_t*)result + size), size));
4840 void release_segment (heap_segment* sg)
4842 ptrdiff_t delta = 0;
4843 FIRE_EVENT(GCFreeSegment_V1, heap_segment_mem(sg));
4844 virtual_free (sg, (uint8_t*)heap_segment_reserved (sg)-(uint8_t*)sg);
4847 heap_segment* gc_heap::get_segment_for_loh (size_t size
4848 #ifdef MULTIPLE_HEAPS
4850 #endif //MULTIPLE_HEAPS
4853 #ifndef MULTIPLE_HEAPS
4855 #endif //MULTIPLE_HEAPS
4856 heap_segment* res = hp->get_segment (size, TRUE);
4859 #ifdef MULTIPLE_HEAPS
4860 heap_segment_heap (res) = hp;
4861 #endif //MULTIPLE_HEAPS
4862 res->flags |= heap_segment_flags_loh;
4864 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(res), (size_t)(heap_segment_reserved (res) - heap_segment_mem(res)), gc_etw_segment_large_object_heap);
4866 GCToEEInterface::DiagUpdateGenerationBounds();
4868 #ifdef MULTIPLE_HEAPS
4869 hp->thread_loh_segment (res);
4871 thread_loh_segment (res);
4872 #endif //MULTIPLE_HEAPS
4878 void gc_heap::thread_loh_segment (heap_segment* new_seg)
4880 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
4882 while (heap_segment_next_rw (seg))
4883 seg = heap_segment_next_rw (seg);
4884 heap_segment_next (seg) = new_seg;
4888 gc_heap::get_large_segment (size_t size, BOOL* did_full_compact_gc)
4890 *did_full_compact_gc = FALSE;
4891 size_t last_full_compact_gc_count = get_full_compact_gc_count();
4893 //access to get_segment needs to be serialized
4894 add_saved_spinlock_info (me_release, mt_get_large_seg);
4896 dprintf (SPINLOCK_LOG, ("[%d]Seg: Lmsl", heap_number));
4897 leave_spin_lock (&more_space_lock);
4898 enter_spin_lock (&gc_heap::gc_lock);
4899 dprintf (SPINLOCK_LOG, ("[%d]Seg: Egc", heap_number));
4900 // if a GC happened between here and before we ask for a segment in
4901 // get_large_segment, we need to count that GC.
4902 size_t current_full_compact_gc_count = get_full_compact_gc_count();
4904 if (current_full_compact_gc_count > last_full_compact_gc_count)
4906 *did_full_compact_gc = TRUE;
4909 #ifdef BACKGROUND_GC
4910 while (current_c_gc_state == c_gc_state_planning)
4912 dprintf (3, ("lh state planning, waiting to get a large seg"));
4914 dprintf (SPINLOCK_LOG, ("[%d]Seg: P, Lgc", heap_number));
4915 leave_spin_lock (&gc_lock);
4916 background_gc_wait_lh (awr_get_loh_seg);
4917 enter_spin_lock (&gc_lock);
4918 dprintf (SPINLOCK_LOG, ("[%d]Seg: P, Egc", heap_number));
4920 assert ((current_c_gc_state == c_gc_state_free) ||
4921 (current_c_gc_state == c_gc_state_marking));
4922 #endif //BACKGROUND_GC
4924 heap_segment* res = get_segment_for_loh (size
4925 #ifdef MULTIPLE_HEAPS
4927 #endif //MULTIPLE_HEAPS
4930 dprintf (SPINLOCK_LOG, ("[%d]Seg: A Lgc", heap_number));
4931 leave_spin_lock (&gc_heap::gc_lock);
4932 enter_spin_lock (&more_space_lock);
4933 dprintf (SPINLOCK_LOG, ("[%d]Seg: A Emsl", heap_number));
4934 add_saved_spinlock_info (me_acquire, mt_get_large_seg);
4936 #ifdef BACKGROUND_GC
4937 wait_for_background_planning (awr_get_loh_seg);
4938 #endif //BACKGROUND_GC
4944 BOOL gc_heap::unprotect_segment (heap_segment* seg)
4946 uint8_t* start = align_lower_page (heap_segment_mem (seg));
4947 ptrdiff_t region_size = heap_segment_allocated (seg) - start;
4949 if (region_size != 0 )
4951 dprintf (3, ("unprotecting segment %Ix:", (size_t)seg));
4953 BOOL status = GCToOSInterface::VirtualUnprotect (start, region_size);
4961 #ifdef MULTIPLE_HEAPS
4964 #pragma warning(disable:4035)
4965 static ptrdiff_t get_cycle_count()
4969 #pragma warning(default:4035)
4970 #elif defined(__GNUC__)
4971 static ptrdiff_t get_cycle_count()
4975 __asm__ __volatile__
4976 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
4980 #error Unknown compiler
4982 #elif defined(_TARGET_AMD64_)
4984 extern "C" uint64_t __rdtsc();
4985 #pragma intrinsic(__rdtsc)
4986 static ptrdiff_t get_cycle_count()
4988 return (ptrdiff_t)__rdtsc();
4990 #elif defined(__clang__)
4991 static ptrdiff_t get_cycle_count()
4995 __asm__ __volatile__
4996 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
4997 return (cyclesHi << 32) | cycles;
5000 extern "C" ptrdiff_t get_cycle_count(void);
5002 #elif defined(_TARGET_ARM_)
5003 static ptrdiff_t get_cycle_count()
5005 // @ARMTODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
5006 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
5007 // all buffer access times being reported as equal in access_time().
5010 #elif defined(_TARGET_ARM64_)
5011 static ptrdiff_t get_cycle_count()
5013 // @ARM64TODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
5014 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
5015 // all buffer access times being reported as equal in access_time().
5019 #error NYI platform: get_cycle_count
5020 #endif //_TARGET_X86_
5025 static uint8_t* sniff_buffer;
5026 static unsigned n_sniff_buffers;
5027 static unsigned cur_sniff_index;
5029 static uint16_t proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5030 static uint16_t heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5031 static uint16_t heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5032 static uint16_t heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5033 static uint16_t heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5034 static uint16_t numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5036 static int access_time(uint8_t *sniff_buffer, int heap_number, unsigned sniff_index, unsigned n_sniff_buffers)
5038 ptrdiff_t start_cycles = get_cycle_count();
5039 uint8_t sniff = sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE];
5040 assert (sniff == 0);
5041 ptrdiff_t elapsed_cycles = get_cycle_count() - start_cycles;
5042 // add sniff here just to defeat the optimizer
5043 elapsed_cycles += sniff;
5044 return (int) elapsed_cycles;
5048 static BOOL init(int n_heaps)
5050 assert (sniff_buffer == NULL && n_sniff_buffers == 0);
5051 if (!GCToOSInterface::CanGetCurrentProcessorNumber())
5053 n_sniff_buffers = n_heaps*2+1;
5054 size_t n_cache_lines = 1 + n_heaps * n_sniff_buffers + 1;
5055 size_t sniff_buf_size = n_cache_lines * HS_CACHE_LINE_SIZE;
5056 if (sniff_buf_size / HS_CACHE_LINE_SIZE != n_cache_lines) // check for overlow
5061 sniff_buffer = new (nothrow) uint8_t[sniff_buf_size];
5062 if (sniff_buffer == 0)
5064 memset(sniff_buffer, 0, sniff_buf_size*sizeof(uint8_t));
5067 //can not enable gc numa aware, force all heaps to be in
5068 //one numa node by filling the array with all 0s
5069 if (!GCToOSInterface::CanEnableGCNumaAware())
5070 memset(heap_no_to_numa_node, 0, sizeof (heap_no_to_numa_node));
5075 static void init_cpu_mapping(gc_heap * /*heap*/, int heap_number)
5077 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5079 uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps;
5080 // We can safely cast heap_number to a uint16_t 'cause GetCurrentProcessCpuCount
5081 // only returns up to MAX_SUPPORTED_CPUS procs right now. We only ever create at most
5082 // MAX_SUPPORTED_CPUS GC threads.
5083 proc_no_to_heap_no[proc_no] = (uint16_t)heap_number;
5087 static void mark_heap(int heap_number)
5089 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5092 for (unsigned sniff_index = 0; sniff_index < n_sniff_buffers; sniff_index++)
5093 sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5096 static int select_heap(alloc_context* acontext, int /*hint*/)
5098 UNREFERENCED_PARAMETER(acontext); // only referenced by dprintf
5100 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5101 return proc_no_to_heap_no[GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps];
5103 unsigned sniff_index = Interlocked::Increment(&cur_sniff_index);
5104 sniff_index %= n_sniff_buffers;
5107 int best_access_time = 1000*1000*1000;
5108 int second_best_access_time = best_access_time;
5110 uint8_t *l_sniff_buffer = sniff_buffer;
5111 unsigned l_n_sniff_buffers = n_sniff_buffers;
5112 for (int heap_number = 0; heap_number < gc_heap::n_heaps; heap_number++)
5114 int this_access_time = access_time(l_sniff_buffer, heap_number, sniff_index, l_n_sniff_buffers);
5115 if (this_access_time < best_access_time)
5117 second_best_access_time = best_access_time;
5118 best_access_time = this_access_time;
5119 best_heap = heap_number;
5121 else if (this_access_time < second_best_access_time)
5123 second_best_access_time = this_access_time;
5127 if (best_access_time*2 < second_best_access_time)
5129 sniff_buffer[(1 + best_heap*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5131 dprintf (3, ("select_heap yields crisp %d for context %p\n", best_heap, (void *)acontext));
5135 dprintf (3, ("select_heap yields vague %d for context %p\n", best_heap, (void *)acontext ));
5141 static bool can_find_heap_fast()
5143 return GCToOSInterface::CanGetCurrentProcessorNumber();
5146 static uint16_t find_proc_no_from_heap_no(int heap_number)
5148 return heap_no_to_proc_no[heap_number];
5151 static void set_proc_no_for_heap(int heap_number, uint16_t proc_no)
5153 heap_no_to_proc_no[heap_number] = proc_no;
5156 static uint16_t find_numa_node_from_heap_no(int heap_number)
5158 return heap_no_to_numa_node[heap_number];
5161 static void set_numa_node_for_heap(int heap_number, uint16_t numa_node)
5163 heap_no_to_numa_node[heap_number] = numa_node;
5166 static uint16_t find_cpu_group_from_heap_no(int heap_number)
5168 return heap_no_to_cpu_group[heap_number];
5171 static void set_cpu_group_for_heap(int heap_number, uint16_t group_number)
5173 heap_no_to_cpu_group[heap_number] = group_number;
5176 static uint16_t find_group_proc_from_heap_no(int heap_number)
5178 return heap_no_to_group_proc[heap_number];
5181 static void set_group_proc_for_heap(int heap_number, uint16_t group_proc)
5183 heap_no_to_group_proc[heap_number] = group_proc;
5186 static void init_numa_node_to_heap_map(int nheaps)
5187 { // called right after GCHeap::Init() for each heap is finished
5188 // when numa is not enabled, heap_no_to_numa_node[] are all filled
5189 // with 0s during initialization, and will be treated as one node
5190 numa_node_to_heap_map[0] = 0;
5193 for (int i=1; i < nheaps; i++)
5195 if (heap_no_to_numa_node[i] != heap_no_to_numa_node[i-1])
5196 numa_node_to_heap_map[node_index++] = (uint16_t)i;
5198 numa_node_to_heap_map[node_index] = (uint16_t)nheaps; //mark the end with nheaps
5201 static void get_heap_range_for_heap(int hn, int* start, int* end)
5202 { // 1-tier/no numa case: heap_no_to_numa_node[] all zeros,
5203 // and treated as in one node. thus: start=0, end=n_heaps
5204 uint16_t numa_node = heap_no_to_numa_node[hn];
5205 *start = (int)numa_node_to_heap_map[numa_node];
5206 *end = (int)(numa_node_to_heap_map[numa_node+1]);
5209 uint8_t* heap_select::sniff_buffer;
5210 unsigned heap_select::n_sniff_buffers;
5211 unsigned heap_select::cur_sniff_index;
5212 uint16_t heap_select::proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5213 uint16_t heap_select::heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5214 uint16_t heap_select::heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5215 uint16_t heap_select::heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5216 uint16_t heap_select::heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5217 uint16_t heap_select::numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5219 BOOL gc_heap::create_thread_support (unsigned number_of_heaps)
5222 if (!gc_start_event.CreateOSManualEventNoThrow (FALSE))
5226 if (!ee_suspend_event.CreateOSAutoEventNoThrow (FALSE))
5230 if (!gc_t_join.init (number_of_heaps, join_flavor_server_gc))
5241 destroy_thread_support();
5247 void gc_heap::destroy_thread_support ()
5249 if (ee_suspend_event.IsValid())
5251 ee_suspend_event.CloseEvent();
5253 if (gc_start_event.IsValid())
5255 gc_start_event.CloseEvent();
5259 void set_thread_group_affinity_for_heap(int heap_number, GCThreadAffinity* affinity)
5261 affinity->Group = GCThreadAffinity::None;
5262 affinity->Processor = GCThreadAffinity::None;
5265 GCToOSInterface::GetGroupForProcessor((uint16_t)heap_number, &gn, &gpn);
5268 for (uintptr_t mask = 1; mask !=0; mask <<=1)
5270 if (bit_number == gpn)
5272 dprintf(3, ("using processor group %d, mask %Ix for heap %d\n", gn, mask, heap_number));
5273 affinity->Processor = gpn;
5274 affinity->Group = gn;
5275 heap_select::set_cpu_group_for_heap(heap_number, gn);
5276 heap_select::set_group_proc_for_heap(heap_number, gpn);
5277 if (GCToOSInterface::CanEnableGCNumaAware())
5279 PROCESSOR_NUMBER proc_no;
5281 proc_no.Number = (uint8_t)gpn;
5282 proc_no.Reserved = 0;
5284 uint16_t node_no = 0;
5285 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5286 heap_select::set_numa_node_for_heap(heap_number, node_no);
5289 { // no numa setting, each cpu group is treated as a node
5290 heap_select::set_numa_node_for_heap(heap_number, gn);
5298 void set_thread_affinity_mask_for_heap(int heap_number, GCThreadAffinity* affinity)
5300 affinity->Group = GCThreadAffinity::None;
5301 affinity->Processor = GCThreadAffinity::None;
5303 uintptr_t pmask, smask;
5304 if (GCToOSInterface::GetCurrentProcessAffinityMask(&pmask, &smask))
5308 uint8_t proc_number = 0;
5309 for (uintptr_t mask = 1; mask != 0; mask <<= 1)
5311 if ((mask & pmask) != 0)
5313 if (bit_number == heap_number)
5315 dprintf (3, ("Using processor %d for heap %d", proc_number, heap_number));
5316 affinity->Processor = proc_number;
5317 heap_select::set_proc_no_for_heap(heap_number, proc_number);
5318 if (GCToOSInterface::CanEnableGCNumaAware())
5320 uint16_t node_no = 0;
5321 PROCESSOR_NUMBER proc_no;
5323 proc_no.Number = (uint8_t)proc_number;
5324 proc_no.Reserved = 0;
5325 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5327 heap_select::set_numa_node_for_heap(heap_number, node_no);
5339 bool gc_heap::create_gc_thread ()
5341 dprintf (3, ("Creating gc thread\n"));
5342 return GCToEEInterface::CreateThread(gc_thread_stub, this, false, ".NET Server GC");
5346 #pragma warning(disable:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5348 void gc_heap::gc_thread_function ()
5350 assert (gc_done_event.IsValid());
5351 assert (gc_start_event.IsValid());
5352 dprintf (3, ("gc thread started"));
5354 heap_select::init_cpu_mapping(this, heap_number);
5358 assert (!gc_t_join.joined());
5360 if (heap_number == 0)
5362 gc_heap::ee_suspend_event.Wait(INFINITE, FALSE);
5364 BEGIN_TIMING(suspend_ee_during_log);
5365 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
5366 END_TIMING(suspend_ee_during_log);
5368 proceed_with_gc_p = TRUE;
5370 if (!should_proceed_with_gc())
5372 update_collection_counts_for_no_gc();
5373 proceed_with_gc_p = FALSE;
5377 settings.init_mechanisms();
5378 gc_start_event.Set();
5380 dprintf (3, ("%d gc thread waiting...", heap_number));
5384 gc_start_event.Wait(INFINITE, FALSE);
5385 dprintf (3, ("%d gc thread waiting... Done", heap_number));
5388 assert ((heap_number == 0) || proceed_with_gc_p);
5390 if (proceed_with_gc_p)
5391 garbage_collect (GCHeap::GcCondemnedGeneration);
5393 if (heap_number == 0)
5395 if (proceed_with_gc_p && (!settings.concurrent))
5400 #ifdef BACKGROUND_GC
5401 recover_bgc_settings();
5402 #endif //BACKGROUND_GC
5404 #ifdef MULTIPLE_HEAPS
5405 for (int i = 0; i < gc_heap::n_heaps; i++)
5407 gc_heap* hp = gc_heap::g_heaps[i];
5408 hp->add_saved_spinlock_info (me_release, mt_block_gc);
5409 dprintf (SPINLOCK_LOG, ("[%d]GC Lmsl", i));
5410 leave_spin_lock(&hp->more_space_lock);
5412 #endif //MULTIPLE_HEAPS
5414 gc_heap::gc_started = FALSE;
5416 BEGIN_TIMING(restart_ee_during_log);
5417 GCToEEInterface::RestartEE(TRUE);
5418 END_TIMING(restart_ee_during_log);
5419 process_sync_log_stats();
5421 dprintf (SPINLOCK_LOG, ("GC Lgc"));
5422 leave_spin_lock (&gc_heap::gc_lock);
5424 gc_heap::internal_gc_done = true;
5426 if (proceed_with_gc_p)
5430 // If we didn't actually do a GC, it means we didn't wait up the other threads,
5431 // we still need to set the gc_done_event for those threads.
5432 for (int i = 0; i < gc_heap::n_heaps; i++)
5434 gc_heap* hp = gc_heap::g_heaps[i];
5441 int spin_count = 32 * (gc_heap::n_heaps - 1);
5443 // wait until RestartEE has progressed to a stage where we can restart user threads
5444 while (!gc_heap::internal_gc_done && !GCHeap::SafeToRestartManagedThreads())
5446 spin_and_switch (spin_count, (gc_heap::internal_gc_done || GCHeap::SafeToRestartManagedThreads()));
5453 #pragma warning(default:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5456 #endif //MULTIPLE_HEAPS
5458 bool virtual_alloc_commit_for_heap(void* addr, size_t size, int h_number)
5460 #if defined(MULTIPLE_HEAPS) && !defined(FEATURE_REDHAWK)
5461 // Currently there is no way for us to specific the numa node to allocate on via hosting interfaces to
5462 // a host. This will need to be added later.
5463 #if !defined(FEATURE_CORECLR)
5464 if (!CLRMemoryHosted())
5467 if (GCToOSInterface::CanEnableGCNumaAware())
5469 uint32_t numa_node = heap_select::find_numa_node_from_heap_no(h_number);
5470 if (GCToOSInterface::VirtualCommit(addr, size, numa_node))
5475 UNREFERENCED_PARAMETER(h_number);
5478 //numa aware not enabled, or call failed --> fallback to VirtualCommit()
5479 return GCToOSInterface::VirtualCommit(addr, size);
5482 #ifndef SEG_MAPPING_TABLE
5484 heap_segment* gc_heap::segment_of (uint8_t* add, ptrdiff_t& delta, BOOL verify_p)
5486 uint8_t* sadd = add;
5487 heap_segment* hs = 0;
5488 heap_segment* hs1 = 0;
5489 if (!((add >= g_gc_lowest_address) && (add < g_gc_highest_address)))
5494 //repeat in case there is a concurrent insertion in the table.
5499 seg_table->lookup (sadd);
5500 hs1 = (heap_segment*)sadd;
5501 } while (hs1 && !in_range_for_segment (add, hs1) && (hs != hs1));
5506 (verify_p && (add > heap_segment_reserved ((heap_segment*)(sadd + delta)))))
5510 #endif //SEG_MAPPING_TABLE
5518 // If we want to save space we can have a pool of plug_and_gap's instead of
5519 // always having 2 allocated for each pinned plug.
5520 gap_reloc_pair saved_pre_plug;
5521 // If we decide to not compact, we need to restore the original values.
5522 gap_reloc_pair saved_pre_plug_reloc;
5524 gap_reloc_pair saved_post_plug;
5526 // Supposedly Pinned objects cannot have references but we are seeing some from pinvoke
5527 // frames. Also if it's an artificially pinned plug created by us, it can certainly
5529 // We know these cases will be rare so we can optimize this to be only allocated on decommand.
5530 gap_reloc_pair saved_post_plug_reloc;
5532 // We need to calculate this after we are done with plan phase and before compact
5533 // phase because compact phase will change the bricks so relocate_address will no
5535 uint8_t* saved_pre_plug_info_reloc_start;
5537 // We need to save this because we will have no way to calculate it, unlike the
5538 // pre plug info start which is right before this plug.
5539 uint8_t* saved_post_plug_info_start;
5542 uint8_t* allocation_context_start_region;
5543 #endif //SHORT_PLUGS
5545 // How the bits in these bytes are organized:
5547 // bit to indicate whether it's a short obj | 3 bits for refs in this short obj | 2 unused bits | bit to indicate if it's collectible | last bit
5548 // last bit indicates if there's pre or post info associated with this plug. If it's not set all other bits will be 0.
5553 // We are seeing this is getting corrupted for a PP with a NP after.
5554 // Save it when we first set it and make sure it doesn't change.
5555 gap_reloc_pair saved_post_plug_debug;
5558 size_t get_max_short_bits()
5560 return (sizeof (gap_reloc_pair) / sizeof (uint8_t*));
5564 size_t get_pre_short_start_bit ()
5566 return (sizeof (saved_pre_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5571 return (saved_pre_p & (1 << (sizeof (saved_pre_p) * 8 - 1)));
5574 void set_pre_short()
5576 saved_pre_p |= (1 << (sizeof (saved_pre_p) * 8 - 1));
5579 void set_pre_short_bit (size_t bit)
5581 saved_pre_p |= 1 << (get_pre_short_start_bit() + bit);
5584 BOOL pre_short_bit_p (size_t bit)
5586 return (saved_pre_p & (1 << (get_pre_short_start_bit() + bit)));
5589 #ifdef COLLECTIBLE_CLASS
5590 void set_pre_short_collectible()
5595 BOOL pre_short_collectible_p()
5597 return (saved_pre_p & 2);
5599 #endif //COLLECTIBLE_CLASS
5602 size_t get_post_short_start_bit ()
5604 return (sizeof (saved_post_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5609 return (saved_post_p & (1 << (sizeof (saved_post_p) * 8 - 1)));
5612 void set_post_short()
5614 saved_post_p |= (1 << (sizeof (saved_post_p) * 8 - 1));
5617 void set_post_short_bit (size_t bit)
5619 saved_post_p |= 1 << (get_post_short_start_bit() + bit);
5622 BOOL post_short_bit_p (size_t bit)
5624 return (saved_post_p & (1 << (get_post_short_start_bit() + bit)));
5627 #ifdef COLLECTIBLE_CLASS
5628 void set_post_short_collectible()
5633 BOOL post_short_collectible_p()
5635 return (saved_post_p & 2);
5637 #endif //COLLECTIBLE_CLASS
5639 uint8_t* get_plug_address() { return first; }
5641 BOOL has_pre_plug_info() { return saved_pre_p; }
5642 BOOL has_post_plug_info() { return saved_post_p; }
5644 gap_reloc_pair* get_pre_plug_reloc_info() { return &saved_pre_plug_reloc; }
5645 gap_reloc_pair* get_post_plug_reloc_info() { return &saved_post_plug_reloc; }
5646 void set_pre_plug_info_reloc_start (uint8_t* reloc) { saved_pre_plug_info_reloc_start = reloc; }
5647 uint8_t* get_post_plug_info_start() { return saved_post_plug_info_start; }
5649 // We need to temporarily recover the shortened plugs for compact phase so we can
5650 // copy over the whole plug and their related info (mark bits/cards). But we will
5651 // need to set the artificial gap back so compact phase can keep reading the plug info.
5652 // We also need to recover the saved info because we'll need to recover it later.
5654 // So we would call swap_p*_plug_and_saved once to recover the object info; then call
5655 // it again to recover the artificial gap.
5656 void swap_pre_plug_and_saved()
5658 gap_reloc_pair temp;
5659 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5660 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5661 saved_pre_plug_reloc = temp;
5664 void swap_post_plug_and_saved()
5666 gap_reloc_pair temp;
5667 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5668 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5669 saved_post_plug_reloc = temp;
5672 void swap_pre_plug_and_saved_for_profiler()
5674 gap_reloc_pair temp;
5675 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5676 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5677 saved_pre_plug = temp;
5680 void swap_post_plug_and_saved_for_profiler()
5682 gap_reloc_pair temp;
5683 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5684 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5685 saved_post_plug = temp;
5688 // We should think about whether it's really necessary to have to copy back the pre plug
5689 // info since it was already copied during compacting plugs. But if a plug doesn't move
5690 // by >= 3 ptr size (the size of gap_reloc_pair), it means we'd have to recover pre plug info.
5691 void recover_plug_info()
5695 if (gc_heap::settings.compaction)
5697 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5699 &saved_pre_plug_reloc,
5700 saved_pre_plug_info_reloc_start));
5701 memcpy (saved_pre_plug_info_reloc_start, &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5705 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5708 (first - sizeof (plug_and_gap))));
5709 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5715 if (gc_heap::settings.compaction)
5717 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5719 &saved_post_plug_reloc,
5720 saved_post_plug_info_start));
5721 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5725 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5728 saved_post_plug_info_start));
5729 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5736 void gc_mechanisms::init_mechanisms()
5738 condemned_generation = 0;
5739 promotion = FALSE;//TRUE;
5741 #ifdef FEATURE_LOH_COMPACTION
5742 loh_compaction = gc_heap::should_compact_loh();
5744 loh_compaction = FALSE;
5745 #endif //FEATURE_LOH_COMPACTION
5746 heap_expansion = FALSE;
5749 elevation_reduced = FALSE;
5750 found_finalizers = FALSE;
5751 #ifdef BACKGROUND_GC
5752 background_p = recursive_gc_sync::background_running_p() != FALSE;
5753 allocations_allowed = TRUE;
5754 #endif //BACKGROUND_GC
5756 entry_memory_load = 0;
5757 exit_memory_load = 0;
5760 stress_induced = FALSE;
5761 #endif // STRESS_HEAP
5764 void gc_mechanisms::first_init()
5767 gen0_reduction_count = 0;
5768 should_lock_elevation = FALSE;
5769 elevation_locked_count = 0;
5770 reason = reason_empty;
5771 #ifdef BACKGROUND_GC
5772 pause_mode = gc_heap::gc_can_use_concurrent ? pause_interactive : pause_batch;
5774 int debug_pause_mode = static_cast<int>(GCConfig::GetLatencyMode());
5775 if (debug_pause_mode >= 0)
5777 assert (debug_pause_mode <= pause_sustained_low_latency);
5778 pause_mode = (gc_pause_mode)debug_pause_mode;
5781 #else //BACKGROUND_GC
5782 pause_mode = pause_batch;
5783 #endif //BACKGROUND_GC
5788 void gc_mechanisms::record (gc_history_global* history)
5790 #ifdef MULTIPLE_HEAPS
5791 history->num_heaps = gc_heap::n_heaps;
5793 history->num_heaps = 1;
5794 #endif //MULTIPLE_HEAPS
5796 history->condemned_generation = condemned_generation;
5797 history->gen0_reduction_count = gen0_reduction_count;
5798 history->reason = reason;
5799 history->pause_mode = (int)pause_mode;
5800 history->mem_pressure = entry_memory_load;
5801 history->global_mechanims_p = 0;
5803 // start setting the boolean values.
5805 history->set_mechanism_p (global_concurrent);
5808 history->set_mechanism_p (global_compaction);
5811 history->set_mechanism_p (global_promotion);
5814 history->set_mechanism_p (global_demotion);
5817 history->set_mechanism_p (global_card_bundles);
5819 if (elevation_reduced)
5820 history->set_mechanism_p (global_elevation);
5823 /**********************************
5824 called at the beginning of GC to fix the allocated size to
5825 what is really allocated, or to turn the free area into an unused object
5826 It needs to be called after all of the other allocation contexts have been
5827 fixed since it relies on alloc_allocated.
5828 ********************************/
5830 //for_gc_p indicates that the work is being done for GC,
5831 //as opposed to concurrent heap verification
5832 void gc_heap::fix_youngest_allocation_area (BOOL for_gc_p)
5834 UNREFERENCED_PARAMETER(for_gc_p);
5836 // The gen 0 alloc context is never used for allocation in the allocator path. It's
5837 // still used in the allocation path during GCs.
5838 assert (generation_allocation_pointer (youngest_generation) == nullptr);
5839 assert (generation_allocation_limit (youngest_generation) == nullptr);
5840 heap_segment_allocated (ephemeral_heap_segment) = alloc_allocated;
5843 void gc_heap::fix_large_allocation_area (BOOL for_gc_p)
5845 UNREFERENCED_PARAMETER(for_gc_p);
5848 alloc_context* acontext =
5850 generation_alloc_context (large_object_generation);
5851 assert (acontext->alloc_ptr == 0);
5852 assert (acontext->alloc_limit == 0);
5854 dprintf (3, ("Large object alloc context: ptr: %Ix, limit %Ix",
5855 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5856 fix_allocation_context (acontext, FALSE, get_alignment_constant (FALSE));
5859 acontext->alloc_ptr = 0;
5860 acontext->alloc_limit = acontext->alloc_ptr;
5865 //for_gc_p indicates that the work is being done for GC,
5866 //as opposed to concurrent heap verification
5867 void gc_heap::fix_allocation_context (alloc_context* acontext, BOOL for_gc_p,
5870 dprintf (3, ("Fixing allocation context %Ix: ptr: %Ix, limit: %Ix",
5872 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5874 if (((size_t)(alloc_allocated - acontext->alloc_limit) > Align (min_obj_size, align_const)) ||
5877 uint8_t* point = acontext->alloc_ptr;
5880 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
5881 // the allocation area was from the free list
5882 // it was shortened by Align (min_obj_size) to make room for
5883 // at least the shortest unused object
5884 size += Align (min_obj_size, align_const);
5885 assert ((size >= Align (min_obj_size)));
5887 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point,
5888 (size_t)point + size ));
5889 make_unused_array (point, size);
5893 generation_free_obj_space (generation_of (0)) += size;
5894 alloc_contexts_used ++;
5900 alloc_allocated = acontext->alloc_ptr;
5901 assert (heap_segment_allocated (ephemeral_heap_segment) <=
5902 heap_segment_committed (ephemeral_heap_segment));
5903 alloc_contexts_used ++;
5908 // We need to update the alloc_bytes to reflect the portion that we have not used
5909 acontext->alloc_bytes -= (acontext->alloc_limit - acontext->alloc_ptr);
5910 acontext->alloc_ptr = 0;
5911 acontext->alloc_limit = acontext->alloc_ptr;
5915 //used by the heap verification for concurrent gc.
5916 //it nulls out the words set by fix_allocation_context for heap_verification
5917 void repair_allocation (gc_alloc_context* acontext, void*)
5919 uint8_t* point = acontext->alloc_ptr;
5923 dprintf (3, ("Clearing [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
5924 (size_t)acontext->alloc_limit+Align(min_obj_size)));
5925 memclr (acontext->alloc_ptr - plug_skew,
5926 (acontext->alloc_limit - acontext->alloc_ptr)+Align (min_obj_size));
5930 void void_allocation (gc_alloc_context* acontext, void*)
5932 uint8_t* point = acontext->alloc_ptr;
5936 dprintf (3, ("Void [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
5937 (size_t)acontext->alloc_limit+Align(min_obj_size)));
5938 acontext->alloc_ptr = 0;
5939 acontext->alloc_limit = acontext->alloc_ptr;
5943 void gc_heap::repair_allocation_contexts (BOOL repair_p)
5945 GCToEEInterface::GcEnumAllocContexts (repair_p ? repair_allocation : void_allocation, NULL);
5948 struct fix_alloc_context_args
5954 void fix_alloc_context(gc_alloc_context* acontext, void* param)
5956 fix_alloc_context_args* args = (fix_alloc_context_args*)param;
5957 g_theGCHeap->FixAllocContext(acontext, false, (void*)(size_t)(args->for_gc_p), args->heap);
5960 void gc_heap::fix_allocation_contexts(BOOL for_gc_p)
5962 fix_alloc_context_args args;
5963 args.for_gc_p = for_gc_p;
5966 GCToEEInterface::GcEnumAllocContexts(fix_alloc_context, &args);
5967 fix_youngest_allocation_area(for_gc_p);
5968 fix_large_allocation_area(for_gc_p);
5971 void gc_heap::fix_older_allocation_area (generation* older_gen)
5973 heap_segment* older_gen_seg = generation_allocation_segment (older_gen);
5974 if (generation_allocation_limit (older_gen) !=
5975 heap_segment_plan_allocated (older_gen_seg))
5977 uint8_t* point = generation_allocation_pointer (older_gen);
5979 size_t size = (generation_allocation_limit (older_gen) -
5980 generation_allocation_pointer (older_gen));
5983 assert ((size >= Align (min_obj_size)));
5984 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point, (size_t)point+size));
5985 make_unused_array (point, size);
5990 assert (older_gen_seg != ephemeral_heap_segment);
5991 heap_segment_plan_allocated (older_gen_seg) =
5992 generation_allocation_pointer (older_gen);
5993 generation_allocation_limit (older_gen) =
5994 generation_allocation_pointer (older_gen);
5998 void gc_heap::set_allocation_heap_segment (generation* gen)
6000 uint8_t* p = generation_allocation_start (gen);
6002 heap_segment* seg = generation_allocation_segment (gen);
6003 if (in_range_for_segment (p, seg))
6006 // try ephemeral heap segment in case of heap expansion
6007 seg = ephemeral_heap_segment;
6008 if (!in_range_for_segment (p, seg))
6010 seg = heap_segment_rw (generation_start_segment (gen));
6012 PREFIX_ASSUME(seg != NULL);
6014 while (!in_range_for_segment (p, seg))
6016 seg = heap_segment_next_rw (seg);
6017 PREFIX_ASSUME(seg != NULL);
6021 generation_allocation_segment (gen) = seg;
6024 void gc_heap::reset_allocation_pointers (generation* gen, uint8_t* start)
6027 assert (Align ((size_t)start) == (size_t)start);
6028 generation_allocation_start (gen) = start;
6029 generation_allocation_pointer (gen) = 0;//start + Align (min_obj_size);
6030 generation_allocation_limit (gen) = 0;//generation_allocation_pointer (gen);
6031 set_allocation_heap_segment (gen);
6034 #ifdef BACKGROUND_GC
6035 //TODO BACKGROUND_GC this is for test only
6037 gc_heap::disallow_new_allocation (int gen_number)
6039 UNREFERENCED_PARAMETER(gen_number);
6040 settings.allocations_allowed = FALSE;
6043 gc_heap::allow_new_allocation (int gen_number)
6045 UNREFERENCED_PARAMETER(gen_number);
6046 settings.allocations_allowed = TRUE;
6049 #endif //BACKGROUND_GC
6051 bool gc_heap::new_allocation_allowed (int gen_number)
6053 #ifdef BACKGROUND_GC
6054 //TODO BACKGROUND_GC this is for test only
6055 if (!settings.allocations_allowed)
6057 dprintf (2, ("new allocation not allowed"));
6060 #endif //BACKGROUND_GC
6062 if (dd_new_allocation (dynamic_data_of (gen_number)) < 0)
6064 if (gen_number != 0)
6066 // For LOH we will give it more budget before we try a GC.
6067 if (settings.concurrent)
6069 dynamic_data* dd2 = dynamic_data_of (max_generation + 1 );
6071 if (dd_new_allocation (dd2) <= (ptrdiff_t)(-2 * dd_desired_allocation (dd2)))
6079 #ifndef MULTIPLE_HEAPS
6080 else if ((settings.pause_mode != pause_no_gc) && (gen_number == 0))
6082 dprintf (3, ("evaluating allocation rate"));
6083 dynamic_data* dd0 = dynamic_data_of (0);
6084 if ((allocation_running_amount - dd_new_allocation (dd0)) >
6087 uint32_t ctime = GCToOSInterface::GetLowPrecisionTimeStamp();
6088 if ((ctime - allocation_running_time) > 1000)
6090 dprintf (2, (">1s since last gen0 gc"));
6095 allocation_running_amount = dd_new_allocation (dd0);
6099 #endif //MULTIPLE_HEAPS
6104 ptrdiff_t gc_heap::get_desired_allocation (int gen_number)
6106 return dd_desired_allocation (dynamic_data_of (gen_number));
6110 ptrdiff_t gc_heap::get_new_allocation (int gen_number)
6112 return dd_new_allocation (dynamic_data_of (gen_number));
6115 //return the amount allocated so far in gen_number
6117 ptrdiff_t gc_heap::get_allocation (int gen_number)
6119 dynamic_data* dd = dynamic_data_of (gen_number);
6121 return dd_desired_allocation (dd) - dd_new_allocation (dd);
6125 BOOL grow_mark_stack (mark*& m, size_t& len, size_t init_len)
6127 size_t new_size = max (init_len, 2*len);
6128 mark* tmp = new (nothrow) mark [new_size];
6131 memcpy (tmp, m, len * sizeof (mark));
6139 dprintf (1, ("Failed to allocate %Id bytes for mark stack", (len * sizeof (mark))));
6145 uint8_t* pinned_plug (mark* m)
6151 size_t& pinned_len (mark* m)
6157 void set_new_pin_info (mark* m, uint8_t* pin_free_space_start)
6159 m->len = pinned_plug (m) - pin_free_space_start;
6161 m->allocation_context_start_region = pin_free_space_start;
6162 #endif //SHORT_PLUGS
6167 uint8_t*& pin_allocation_context_start_region (mark* m)
6169 return m->allocation_context_start_region;
6172 uint8_t* get_plug_start_in_saved (uint8_t* old_loc, mark* pinned_plug_entry)
6174 uint8_t* saved_pre_plug_info = (uint8_t*)(pinned_plug_entry->get_pre_plug_reloc_info());
6175 uint8_t* plug_start_in_saved = saved_pre_plug_info + (old_loc - (pinned_plug (pinned_plug_entry) - sizeof (plug_and_gap)));
6176 //dprintf (1, ("detected a very short plug: %Ix before PP %Ix, pad %Ix",
6177 // old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6178 dprintf (1, ("EP: %Ix(%Ix), %Ix", old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6179 return plug_start_in_saved;
6183 void set_padding_in_expand (uint8_t* old_loc,
6184 BOOL set_padding_on_saved_p,
6185 mark* pinned_plug_entry)
6187 if (set_padding_on_saved_p)
6189 set_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6193 set_plug_padded (old_loc);
6198 void clear_padding_in_expand (uint8_t* old_loc,
6199 BOOL set_padding_on_saved_p,
6200 mark* pinned_plug_entry)
6202 if (set_padding_on_saved_p)
6204 clear_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6208 clear_plug_padded (old_loc);
6211 #endif //SHORT_PLUGS
6213 void gc_heap::reset_pinned_queue()
6219 void gc_heap::reset_pinned_queue_bos()
6224 // last_pinned_plug is only for asserting purpose.
6225 void gc_heap::merge_with_last_pinned_plug (uint8_t* last_pinned_plug, size_t plug_size)
6227 if (last_pinned_plug)
6229 mark& last_m = mark_stack_array[mark_stack_tos - 1];
6230 assert (last_pinned_plug == last_m.first);
6231 if (last_m.saved_post_p)
6233 last_m.saved_post_p = FALSE;
6234 dprintf (3, ("setting last plug %Ix post to false", last_m.first));
6235 // We need to recover what the gap has overwritten.
6236 memcpy ((last_m.first + last_m.len - sizeof (plug_and_gap)), &(last_m.saved_post_plug), sizeof (gap_reloc_pair));
6238 last_m.len += plug_size;
6239 dprintf (3, ("recovered the last part of plug %Ix, setting its plug size to %Ix", last_m.first, last_m.len));
6243 void gc_heap::set_allocator_next_pin (uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6245 dprintf (3, ("sanp: ptr: %Ix, limit: %Ix", alloc_pointer, alloc_limit));
6246 dprintf (3, ("oldest %Id: %Ix", mark_stack_bos, pinned_plug (oldest_pin())));
6247 if (!(pinned_plug_que_empty_p()))
6249 mark* oldest_entry = oldest_pin();
6250 uint8_t* plug = pinned_plug (oldest_entry);
6251 if ((plug >= alloc_pointer) && (plug < alloc_limit))
6253 alloc_limit = pinned_plug (oldest_entry);
6254 dprintf (3, ("now setting alloc context: %Ix->%Ix(%Id)",
6255 alloc_pointer, alloc_limit, (alloc_limit - alloc_pointer)));
6260 void gc_heap::set_allocator_next_pin (generation* gen)
6262 dprintf (3, ("SANP: gen%d, ptr; %Ix, limit: %Ix", gen->gen_num, generation_allocation_pointer (gen), generation_allocation_limit (gen)));
6263 if (!(pinned_plug_que_empty_p()))
6265 mark* oldest_entry = oldest_pin();
6266 uint8_t* plug = pinned_plug (oldest_entry);
6267 if ((plug >= generation_allocation_pointer (gen)) &&
6268 (plug < generation_allocation_limit (gen)))
6270 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
6271 dprintf (3, ("SANP: get next pin free space in gen%d for alloc: %Ix->%Ix(%Id)",
6273 generation_allocation_pointer (gen), generation_allocation_limit (gen),
6274 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
6277 assert (!((plug < generation_allocation_pointer (gen)) &&
6278 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
6282 // After we set the info, we increase tos.
6283 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6285 UNREFERENCED_PARAMETER(last_pinned_plug);
6287 mark& m = mark_stack_array[mark_stack_tos];
6288 assert (m.first == last_pinned_plug);
6292 set_allocator_next_pin (alloc_pointer, alloc_limit);
6295 // After we set the info, we increase tos.
6296 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, generation* gen)
6298 UNREFERENCED_PARAMETER(last_pinned_plug);
6300 mark& m = mark_stack_array[mark_stack_tos];
6301 assert (m.first == last_pinned_plug);
6306 // Why are we checking here? gen is never 0.
6309 set_allocator_next_pin (gen);
6313 size_t gc_heap::deque_pinned_plug ()
6315 dprintf (3, ("dequed: %Id", mark_stack_bos));
6316 size_t m = mark_stack_bos;
6322 mark* gc_heap::pinned_plug_of (size_t bos)
6324 return &mark_stack_array [ bos ];
6328 mark* gc_heap::oldest_pin ()
6330 return pinned_plug_of (mark_stack_bos);
6334 BOOL gc_heap::pinned_plug_que_empty_p ()
6336 return (mark_stack_bos == mark_stack_tos);
6340 mark* gc_heap::before_oldest_pin()
6342 if (mark_stack_bos >= 1)
6343 return pinned_plug_of (mark_stack_bos-1);
6349 BOOL gc_heap::ephemeral_pointer_p (uint8_t* o)
6351 return ((o >= ephemeral_low) && (o < ephemeral_high));
6356 int& gc_heap::mark_stack_busy()
6358 return g_mark_stack_busy [(heap_number+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
6362 void gc_heap::make_mark_stack (mark* arr)
6364 reset_pinned_queue();
6365 mark_stack_array = arr;
6366 mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6368 mark_stack_busy() = 0;
6372 #ifdef BACKGROUND_GC
6374 size_t& gc_heap::bpromoted_bytes(int thread)
6376 #ifdef MULTIPLE_HEAPS
6377 return g_bpromoted [thread*16];
6378 #else //MULTIPLE_HEAPS
6379 UNREFERENCED_PARAMETER(thread);
6381 #endif //MULTIPLE_HEAPS
6384 void gc_heap::make_background_mark_stack (uint8_t** arr)
6386 background_mark_stack_array = arr;
6387 background_mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6388 background_mark_stack_tos = arr;
6391 void gc_heap::make_c_mark_list (uint8_t** arr)
6394 c_mark_list_index = 0;
6395 c_mark_list_length = 1 + (OS_PAGE_SIZE / MIN_OBJECT_SIZE);
6397 #endif //BACKGROUND_GC
6402 // The card bundle keeps track of groups of card words.
6403 static const size_t card_bundle_word_width = 32;
6405 // How do we express the fact that 32 bits (card_word_width) is one uint32_t?
6406 static const size_t card_bundle_size = (size_t)(GC_PAGE_SIZE / (sizeof(uint32_t)*card_bundle_word_width));
6409 size_t card_bundle_word (size_t cardb)
6411 return cardb / card_bundle_word_width;
6415 uint32_t card_bundle_bit (size_t cardb)
6417 return (uint32_t)(cardb % card_bundle_word_width);
6420 size_t align_cardw_on_bundle (size_t cardw)
6422 return ((size_t)(cardw + card_bundle_size - 1) & ~(card_bundle_size - 1 ));
6425 // Get the card bundle representing a card word
6426 size_t cardw_card_bundle (size_t cardw)
6428 return cardw / card_bundle_size;
6431 // Get the first card word in a card bundle
6432 size_t card_bundle_cardw (size_t cardb)
6434 return cardb * card_bundle_size;
6437 // Clear the specified card bundle
6438 void gc_heap::card_bundle_clear (size_t cardb)
6440 card_bundle_table [card_bundle_word (cardb)] &= ~(1 << card_bundle_bit (cardb));
6441 dprintf (1,("Cleared card bundle %Ix [%Ix, %Ix[", cardb, (size_t)card_bundle_cardw (cardb),
6442 (size_t)card_bundle_cardw (cardb+1)));
6445 void gc_heap::card_bundle_set (size_t cardb)
6447 if (!card_bundle_set_p (cardb))
6449 card_bundle_table [card_bundle_word (cardb)] |= (1 << card_bundle_bit (cardb));
6453 // Set the card bundle bits between start_cardb and end_cardb
6454 void gc_heap::card_bundles_set (size_t start_cardb, size_t end_cardb)
6456 if (start_cardb == end_cardb)
6458 card_bundle_set(start_cardb);
6462 size_t start_word = card_bundle_word (start_cardb);
6463 size_t end_word = card_bundle_word (end_cardb);
6465 if (start_word < end_word)
6467 // Set the partial words
6468 card_bundle_table [start_word] |= highbits (~0u, card_bundle_bit (start_cardb));
6470 if (card_bundle_bit (end_cardb))
6471 card_bundle_table [end_word] |= lowbits (~0u, card_bundle_bit (end_cardb));
6473 // Set the full words
6474 for (size_t i = start_word + 1; i < end_word; i++)
6475 card_bundle_table [i] = ~0u;
6479 card_bundle_table [start_word] |= (highbits (~0u, card_bundle_bit (start_cardb)) &
6480 lowbits (~0u, card_bundle_bit (end_cardb)));
6484 // Indicates whether the specified bundle is set.
6485 BOOL gc_heap::card_bundle_set_p (size_t cardb)
6487 return (card_bundle_table[card_bundle_word(cardb)] & (1 << card_bundle_bit (cardb)));
6490 // Returns the size (in bytes) of a card bundle representing the region from 'from' to 'end'
6491 size_t size_card_bundle_of (uint8_t* from, uint8_t* end)
6493 // Number of heap bytes represented by a card bundle word
6494 size_t cbw_span = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6496 // Align the start of the region down
6497 from = (uint8_t*)((size_t)from & ~(cbw_span - 1));
6499 // Align the end of the region up
6500 end = (uint8_t*)((size_t)(end + (cbw_span - 1)) & ~(cbw_span - 1));
6502 // Make sure they're really aligned
6503 assert (((size_t)from & (cbw_span - 1)) == 0);
6504 assert (((size_t)end & (cbw_span - 1)) == 0);
6506 return ((end - from) / cbw_span) * sizeof (uint32_t);
6509 // Takes a pointer to a card bundle table and an address, and returns a pointer that represents
6510 // where a theoretical card bundle table that represents every address (starting from 0) would
6511 // start if the bundle word representing the address were to be located at the pointer passed in.
6512 // The returned 'translated' pointer makes it convenient/fast to calculate where the card bundle
6513 // for a given address is using a simple shift operation on the address.
6514 uint32_t* translate_card_bundle_table (uint32_t* cb, uint8_t* lowest_address)
6516 // The number of bytes of heap memory represented by a card bundle word
6517 const size_t heap_bytes_for_bundle_word = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6519 // Each card bundle word is 32 bits
6520 return (uint32_t*)((uint8_t*)cb - (((size_t)lowest_address / heap_bytes_for_bundle_word) * sizeof (uint32_t)));
6523 void gc_heap::enable_card_bundles ()
6525 if (can_use_write_watch_for_card_table() && (!card_bundles_enabled()))
6527 dprintf (1, ("Enabling card bundles"));
6529 // We initially set all of the card bundles
6530 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
6531 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
6532 settings.card_bundles = TRUE;
6536 BOOL gc_heap::card_bundles_enabled ()
6538 return settings.card_bundles;
6541 #endif // CARD_BUNDLE
6543 #if defined (_TARGET_AMD64_)
6544 #define brick_size ((size_t)4096)
6546 #define brick_size ((size_t)2048)
6547 #endif //_TARGET_AMD64_
6550 size_t gc_heap::brick_of (uint8_t* add)
6552 return (size_t)(add - lowest_address) / brick_size;
6556 uint8_t* gc_heap::brick_address (size_t brick)
6558 return lowest_address + (brick_size * brick);
6562 void gc_heap::clear_brick_table (uint8_t* from, uint8_t* end)
6564 for (size_t i = brick_of (from);i < brick_of (end); i++)
6568 //codes for the brick entries:
6569 //entry == 0 -> not assigned
6570 //entry >0 offset is entry-1
6571 //entry <0 jump back entry bricks
6575 void gc_heap::set_brick (size_t index, ptrdiff_t val)
6581 assert (val < 32767);
6583 brick_table [index] = (short)val+1;
6585 brick_table [index] = (short)val;
6589 int gc_heap::get_brick_entry (size_t index)
6591 #ifdef MULTIPLE_HEAPS
6592 return VolatileLoadWithoutBarrier(&brick_table [index]);
6594 return brick_table[index];
6600 uint8_t* align_on_brick (uint8_t* add)
6602 return (uint8_t*)((size_t)(add + brick_size - 1) & ~(brick_size - 1));
6606 uint8_t* align_lower_brick (uint8_t* add)
6608 return (uint8_t*)(((size_t)add) & ~(brick_size - 1));
6611 size_t size_brick_of (uint8_t* from, uint8_t* end)
6613 assert (((size_t)from & (brick_size-1)) == 0);
6614 assert (((size_t)end & (brick_size-1)) == 0);
6616 return ((end - from) / brick_size) * sizeof (short);
6620 uint8_t* gc_heap::card_address (size_t card)
6622 return (uint8_t*) (card_size * card);
6626 size_t gc_heap::card_of ( uint8_t* object)
6628 return (size_t)(object) / card_size;
6632 size_t gc_heap::card_to_brick (size_t card)
6634 return brick_of (card_address (card));
6638 uint8_t* align_on_card (uint8_t* add)
6640 return (uint8_t*)((size_t)(add + card_size - 1) & ~(card_size - 1 ));
6643 uint8_t* align_on_card_word (uint8_t* add)
6645 return (uint8_t*) ((size_t)(add + (card_size*card_word_width)-1) & ~(card_size*card_word_width - 1));
6649 uint8_t* align_lower_card (uint8_t* add)
6651 return (uint8_t*)((size_t)add & ~(card_size-1));
6655 void gc_heap::clear_card (size_t card)
6657 card_table [card_word (card)] =
6658 (card_table [card_word (card)] & ~(1 << card_bit (card)));
6659 dprintf (3,("Cleared card %Ix [%Ix, %Ix[", card, (size_t)card_address (card),
6660 (size_t)card_address (card+1)));
6664 void gc_heap::set_card (size_t card)
6666 size_t word = card_word (card);
6667 card_table[word] = (card_table [word] | (1 << card_bit (card)));
6669 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
6670 // Also set the card bundle that corresponds to the card
6671 size_t bundle_to_set = cardw_card_bundle(word);
6673 card_bundle_set(bundle_to_set);
6675 dprintf (3,("Set card %Ix [%Ix, %Ix[ and bundle %Ix", card, (size_t)card_address (card), (size_t)card_address (card+1), bundle_to_set));
6676 assert(card_bundle_set_p(bundle_to_set) != 0);
6681 BOOL gc_heap::card_set_p (size_t card)
6683 return ( card_table [ card_word (card) ] & (1 << card_bit (card)));
6686 // Returns the number of DWORDs in the card table that cover the
6687 // range of addresses [from, end[.
6688 size_t count_card_of (uint8_t* from, uint8_t* end)
6690 return card_word (gcard_of (end - 1)) - card_word (gcard_of (from)) + 1;
6693 // Returns the number of bytes to allocate for a card table
6694 // that covers the range of addresses [from, end[.
6695 size_t size_card_of (uint8_t* from, uint8_t* end)
6697 return count_card_of (from, end) * sizeof(uint32_t);
6700 // We don't store seg_mapping_table in card_table_info because there's only always one view.
6701 class card_table_info
6705 uint8_t* lowest_address;
6706 uint8_t* highest_address;
6710 uint32_t* card_bundle_table;
6711 #endif //CARD_BUNDLE
6713 // mark_array is always at the end of the data structure because we
6714 // want to be able to make one commit call for everything before it.
6716 uint32_t* mark_array;
6720 uint32_t* next_card_table;
6723 //These are accessors on untranslated cardtable
6725 unsigned& card_table_refcount (uint32_t* c_table)
6727 return *(unsigned*)((char*)c_table - sizeof (card_table_info));
6731 uint8_t*& card_table_lowest_address (uint32_t* c_table)
6733 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->lowest_address;
6736 uint32_t* translate_card_table (uint32_t* ct)
6738 return (uint32_t*)((uint8_t*)ct - card_word (gcard_of (card_table_lowest_address (ct))) * sizeof(uint32_t));
6742 uint8_t*& card_table_highest_address (uint32_t* c_table)
6744 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->highest_address;
6748 short*& card_table_brick_table (uint32_t* c_table)
6750 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->brick_table;
6754 // Get the card bundle table for the specified card table.
6756 uint32_t*& card_table_card_bundle_table (uint32_t* c_table)
6758 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->card_bundle_table;
6760 #endif //CARD_BUNDLE
6763 /* Support for mark_array */
6766 uint32_t*& card_table_mark_array (uint32_t* c_table)
6768 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->mark_array;
6772 #define mark_bit_pitch ((size_t)16)
6774 #define mark_bit_pitch ((size_t)8)
6776 #define mark_word_width ((size_t)32)
6777 #define mark_word_size (mark_word_width * mark_bit_pitch)
6780 uint8_t* align_on_mark_bit (uint8_t* add)
6782 return (uint8_t*)((size_t)(add + (mark_bit_pitch - 1)) & ~(mark_bit_pitch - 1));
6786 uint8_t* align_lower_mark_bit (uint8_t* add)
6788 return (uint8_t*)((size_t)(add) & ~(mark_bit_pitch - 1));
6792 BOOL is_aligned_on_mark_word (uint8_t* add)
6794 return ((size_t)add == ((size_t)(add) & ~(mark_word_size - 1)));
6798 uint8_t* align_on_mark_word (uint8_t* add)
6800 return (uint8_t*)((size_t)(add + mark_word_size - 1) & ~(mark_word_size - 1));
6804 uint8_t* align_lower_mark_word (uint8_t* add)
6806 return (uint8_t*)((size_t)(add) & ~(mark_word_size - 1));
6810 size_t mark_bit_of (uint8_t* add)
6812 return ((size_t)add / mark_bit_pitch);
6816 unsigned int mark_bit_bit (size_t mark_bit)
6818 return (unsigned int)(mark_bit % mark_word_width);
6822 size_t mark_bit_word (size_t mark_bit)
6824 return (mark_bit / mark_word_width);
6828 size_t mark_word_of (uint8_t* add)
6830 return ((size_t)add) / mark_word_size;
6833 uint8_t* mark_word_address (size_t wd)
6835 return (uint8_t*)(wd*mark_word_size);
6838 uint8_t* mark_bit_address (size_t mark_bit)
6840 return (uint8_t*)(mark_bit*mark_bit_pitch);
6844 size_t mark_bit_bit_of (uint8_t* add)
6846 return (((size_t)add / mark_bit_pitch) % mark_word_width);
6850 unsigned int gc_heap::mark_array_marked(uint8_t* add)
6852 return mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add));
6856 BOOL gc_heap::is_mark_bit_set (uint8_t* add)
6858 return (mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add)));
6862 void gc_heap::mark_array_set_marked (uint8_t* add)
6864 size_t index = mark_word_of (add);
6865 uint32_t val = (1 << mark_bit_bit_of (add));
6866 #ifdef MULTIPLE_HEAPS
6867 Interlocked::Or (&(mark_array [index]), val);
6869 mark_array [index] |= val;
6874 void gc_heap::mark_array_clear_marked (uint8_t* add)
6876 mark_array [mark_word_of (add)] &= ~(1 << mark_bit_bit_of (add));
6879 size_t size_mark_array_of (uint8_t* from, uint8_t* end)
6881 assert (((size_t)from & ((mark_word_size)-1)) == 0);
6882 assert (((size_t)end & ((mark_word_size)-1)) == 0);
6883 return sizeof (uint32_t)*(((end - from) / mark_word_size));
6886 //In order to eliminate the lowest_address in the mark array
6887 //computations (mark_word_of, etc) mark_array is offset
6888 // according to the lowest_address.
6889 uint32_t* translate_mark_array (uint32_t* ma)
6891 return (uint32_t*)((uint8_t*)ma - size_mark_array_of (0, g_gc_lowest_address));
6894 // from and end must be page aligned addresses.
6895 void gc_heap::clear_mark_array (uint8_t* from, uint8_t* end, BOOL check_only/*=TRUE*/
6896 #ifdef FEATURE_BASICFREEZE
6897 , BOOL read_only/*=FALSE*/
6898 #endif // FEATURE_BASICFREEZE
6901 if(!gc_can_use_concurrent)
6904 #ifdef FEATURE_BASICFREEZE
6906 #endif // FEATURE_BASICFREEZE
6908 assert (from == align_on_mark_word (from));
6910 assert (end == align_on_mark_word (end));
6912 #ifdef BACKGROUND_GC
6913 uint8_t* current_lowest_address = background_saved_lowest_address;
6914 uint8_t* current_highest_address = background_saved_highest_address;
6916 uint8_t* current_lowest_address = lowest_address;
6917 uint8_t* current_highest_address = highest_address;
6918 #endif //BACKGROUND_GC
6920 //there is a possibility of the addresses to be
6921 //outside of the covered range because of a newly allocated
6922 //large object segment
6923 if ((end <= current_highest_address) && (from >= current_lowest_address))
6925 size_t beg_word = mark_word_of (align_on_mark_word (from));
6926 MAYBE_UNUSED_VAR(beg_word);
6927 //align end word to make sure to cover the address
6928 size_t end_word = mark_word_of (align_on_mark_word (end));
6929 MAYBE_UNUSED_VAR(end_word);
6930 dprintf (3, ("Calling clearing mark array [%Ix, %Ix[ for addresses [%Ix, %Ix[(%s)",
6931 (size_t)mark_word_address (beg_word),
6932 (size_t)mark_word_address (end_word),
6933 (size_t)from, (size_t)end,
6934 (check_only ? "check_only" : "clear")));
6938 while (op < mark_word_address (beg_word))
6940 mark_array_clear_marked (op);
6941 op += mark_bit_pitch;
6944 memset (&mark_array[beg_word], 0, (end_word - beg_word)*sizeof (uint32_t));
6949 //Beware, it is assumed that the mark array word straddling
6950 //start has been cleared before
6951 //verify that the array is empty.
6952 size_t markw = mark_word_of (align_on_mark_word (from));
6953 size_t markw_end = mark_word_of (align_on_mark_word (end));
6954 while (markw < markw_end)
6956 assert (!(mark_array [markw]));
6959 uint8_t* p = mark_word_address (markw_end);
6962 assert (!(mark_array_marked (p)));
6971 //These work on untranslated card tables
6973 uint32_t*& card_table_next (uint32_t* c_table)
6975 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->next_card_table;
6979 size_t& card_table_size (uint32_t* c_table)
6981 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->size;
6984 void own_card_table (uint32_t* c_table)
6986 card_table_refcount (c_table) += 1;
6989 void destroy_card_table (uint32_t* c_table);
6991 void delete_next_card_table (uint32_t* c_table)
6993 uint32_t* n_table = card_table_next (c_table);
6996 if (card_table_next (n_table))
6998 delete_next_card_table (n_table);
7000 if (card_table_refcount (n_table) == 0)
7002 destroy_card_table (n_table);
7003 card_table_next (c_table) = 0;
7008 void release_card_table (uint32_t* c_table)
7010 assert (card_table_refcount (c_table) >0);
7011 card_table_refcount (c_table) -= 1;
7012 if (card_table_refcount (c_table) == 0)
7014 delete_next_card_table (c_table);
7015 if (card_table_next (c_table) == 0)
7017 destroy_card_table (c_table);
7018 // sever the link from the parent
7019 if (&g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))] == c_table)
7021 g_gc_card_table = 0;
7023 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7024 g_gc_card_bundle_table = 0;
7026 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7027 SoftwareWriteWatch::StaticClose();
7028 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7032 uint32_t* p_table = &g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))];
7035 while (p_table && (card_table_next (p_table) != c_table))
7036 p_table = card_table_next (p_table);
7037 card_table_next (p_table) = 0;
7044 void destroy_card_table (uint32_t* c_table)
7046 // delete (uint32_t*)&card_table_refcount(c_table);
7048 GCToOSInterface::VirtualRelease (&card_table_refcount(c_table), card_table_size(c_table));
7049 dprintf (2, ("Table Virtual Free : %Ix", (size_t)&card_table_refcount(c_table)));
7052 uint32_t* gc_heap::make_card_table (uint8_t* start, uint8_t* end)
7054 assert (g_gc_lowest_address == start);
7055 assert (g_gc_highest_address == end);
7057 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7059 size_t bs = size_brick_of (start, end);
7060 size_t cs = size_card_of (start, end);
7062 size_t ms = (gc_can_use_concurrent ?
7063 size_mark_array_of (start, end) :
7072 if (can_use_write_watch_for_card_table())
7074 cb = size_card_bundle_of (g_gc_lowest_address, g_gc_highest_address);
7075 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7076 // If we're not manually managing the card bundles, we will need to use OS write
7077 // watch APIs over this region to track changes.
7078 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7081 #endif //CARD_BUNDLE
7084 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7085 size_t sw_ww_table_offset = 0;
7086 if (gc_can_use_concurrent)
7088 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7089 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7090 wws = sw_ww_table_offset - sw_ww_size_before_table + SoftwareWriteWatch::GetTableByteSize(start, end);
7092 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7094 #ifdef GROWABLE_SEG_MAPPING_TABLE
7095 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7096 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7097 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7099 st += (st_table_offset_aligned - st_table_offset);
7100 #else //GROWABLE_SEG_MAPPING_TABLE
7102 #endif //GROWABLE_SEG_MAPPING_TABLE
7104 // it is impossible for alloc_size to overflow due bounds on each of
7106 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7107 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7112 dprintf (2, ("Init - Card table alloc for %Id bytes: [%Ix, %Ix[",
7113 alloc_size, (size_t)mem, (size_t)(mem+alloc_size)));
7115 // mark array will be committed separately (per segment).
7116 size_t commit_size = alloc_size - ms;
7118 if (!GCToOSInterface::VirtualCommit (mem, commit_size))
7120 dprintf (2, ("Card table commit failed"));
7121 GCToOSInterface::VirtualRelease (mem, alloc_size);
7125 // initialize the ref count
7126 uint32_t* ct = (uint32_t*)(mem+sizeof (card_table_info));
7127 card_table_refcount (ct) = 0;
7128 card_table_lowest_address (ct) = start;
7129 card_table_highest_address (ct) = end;
7130 card_table_brick_table (ct) = (short*)((uint8_t*)ct + cs);
7131 card_table_size (ct) = alloc_size;
7132 card_table_next (ct) = 0;
7135 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7137 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7138 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), g_gc_lowest_address);
7141 #endif //CARD_BUNDLE
7143 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7144 if (gc_can_use_concurrent)
7146 SoftwareWriteWatch::InitializeUntranslatedTable(mem + sw_ww_table_offset, start);
7148 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7150 #ifdef GROWABLE_SEG_MAPPING_TABLE
7151 seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7152 seg_mapping_table = (seg_mapping*)((uint8_t*)seg_mapping_table -
7153 size_seg_mapping_table_of (0, (align_lower_segment (g_gc_lowest_address))));
7154 #endif //GROWABLE_SEG_MAPPING_TABLE
7157 if (gc_can_use_concurrent)
7158 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7160 card_table_mark_array (ct) = NULL;
7163 return translate_card_table(ct);
7166 void gc_heap::set_fgm_result (failure_get_memory f, size_t s, BOOL loh_p)
7168 #ifdef MULTIPLE_HEAPS
7169 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
7171 gc_heap* hp = gc_heap::g_heaps [hn];
7172 hp->fgm_result.set_fgm (f, s, loh_p);
7174 #else //MULTIPLE_HEAPS
7175 fgm_result.set_fgm (f, s, loh_p);
7176 #endif //MULTIPLE_HEAPS
7179 //returns 0 for success, -1 otherwise
7180 // We are doing all the decommitting here because we want to make sure we have
7181 // enough memory to do so - if we do this during copy_brick_card_table and
7182 // and fail to decommit it would make the failure case very complicated to
7183 // handle. This way we can waste some decommit if we call this multiple
7184 // times before the next FGC but it's easier to handle the failure case.
7185 int gc_heap::grow_brick_card_tables (uint8_t* start,
7188 heap_segment* new_seg,
7192 uint8_t* la = g_gc_lowest_address;
7193 uint8_t* ha = g_gc_highest_address;
7194 uint8_t* saved_g_lowest_address = min (start, g_gc_lowest_address);
7195 uint8_t* saved_g_highest_address = max (end, g_gc_highest_address);
7196 seg_mapping* new_seg_mapping_table = nullptr;
7197 #ifdef BACKGROUND_GC
7198 // This value is only for logging purpose - it's not necessarily exactly what we
7199 // would commit for mark array but close enough for diagnostics purpose.
7200 size_t logging_ma_commit_size = size_mark_array_of (0, (uint8_t*)size);
7201 #endif //BACKGROUND_GC
7203 // See if the address is already covered
7204 if ((la != saved_g_lowest_address ) || (ha != saved_g_highest_address))
7207 //modify the higest address so the span covered
7208 //is twice the previous one.
7209 uint8_t* top = (uint8_t*)0 + Align (GCToOSInterface::GetVirtualMemoryLimit());
7210 // On non-Windows systems, we get only an approximate value that can possibly be
7211 // slightly lower than the saved_g_highest_address.
7212 // In such case, we set the top to the saved_g_highest_address so that the
7213 // card and brick tables always cover the whole new range.
7214 if (top < saved_g_highest_address)
7216 top = saved_g_highest_address;
7220 if (ps > (uint64_t)200*1024*1024*1024)
7221 ps += (uint64_t)100*1024*1024*1024;
7226 if (saved_g_lowest_address < g_gc_lowest_address)
7228 if (ps > (size_t)g_gc_lowest_address)
7229 saved_g_lowest_address = (uint8_t*)(size_t)OS_PAGE_SIZE;
7232 assert (((size_t)g_gc_lowest_address - ps) >= OS_PAGE_SIZE);
7233 saved_g_lowest_address = min (saved_g_lowest_address, (g_gc_lowest_address - ps));
7237 if (saved_g_highest_address > g_gc_highest_address)
7239 saved_g_highest_address = max ((saved_g_lowest_address + ps), saved_g_highest_address);
7240 if (saved_g_highest_address > top)
7241 saved_g_highest_address = top;
7244 dprintf (GC_TABLE_LOG, ("Growing card table [%Ix, %Ix[",
7245 (size_t)saved_g_lowest_address,
7246 (size_t)saved_g_highest_address));
7248 bool write_barrier_updated = false;
7249 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7250 uint32_t* saved_g_card_table = g_gc_card_table;
7252 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7253 uint32_t* saved_g_card_bundle_table = g_gc_card_bundle_table;
7257 uint32_t* translated_ct = 0;
7260 size_t cs = size_card_of (saved_g_lowest_address, saved_g_highest_address);
7261 size_t bs = size_brick_of (saved_g_lowest_address, saved_g_highest_address);
7264 size_t ms = (gc_heap::gc_can_use_concurrent ?
7265 size_mark_array_of (saved_g_lowest_address, saved_g_highest_address) :
7274 if (can_use_write_watch_for_card_table())
7276 cb = size_card_bundle_of (saved_g_lowest_address, saved_g_highest_address);
7278 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7279 // If we're not manually managing the card bundles, we will need to use OS write
7280 // watch APIs over this region to track changes.
7281 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7284 #endif //CARD_BUNDLE
7287 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7288 size_t sw_ww_table_offset = 0;
7289 if (gc_can_use_concurrent)
7291 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7292 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7294 sw_ww_table_offset -
7295 sw_ww_size_before_table +
7296 SoftwareWriteWatch::GetTableByteSize(saved_g_lowest_address, saved_g_highest_address);
7298 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7300 #ifdef GROWABLE_SEG_MAPPING_TABLE
7301 size_t st = size_seg_mapping_table_of (saved_g_lowest_address, saved_g_highest_address);
7302 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7303 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7304 st += (st_table_offset_aligned - st_table_offset);
7305 #else //GROWABLE_SEG_MAPPING_TABLE
7307 #endif //GROWABLE_SEG_MAPPING_TABLE
7309 // it is impossible for alloc_size to overflow due bounds on each of
7311 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7312 dprintf (GC_TABLE_LOG, ("card table: %Id; brick table: %Id; card bundle: %Id; sw ww table: %Id; seg table: %Id; mark array: %Id",
7313 cs, bs, cb, wws, st, ms));
7315 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7319 set_fgm_result (fgm_grow_table, alloc_size, loh_p);
7323 dprintf (GC_TABLE_LOG, ("Table alloc for %Id bytes: [%Ix, %Ix[",
7324 alloc_size, (size_t)mem, (size_t)((uint8_t*)mem+alloc_size)));
7327 // mark array will be committed separately (per segment).
7328 size_t commit_size = alloc_size - ms;
7330 if (!GCToOSInterface::VirtualCommit (mem, commit_size))
7332 dprintf (GC_TABLE_LOG, ("Table commit failed"));
7333 set_fgm_result (fgm_commit_table, commit_size, loh_p);
7338 ct = (uint32_t*)(mem + sizeof (card_table_info));
7339 card_table_refcount (ct) = 0;
7340 card_table_lowest_address (ct) = saved_g_lowest_address;
7341 card_table_highest_address (ct) = saved_g_highest_address;
7342 card_table_next (ct) = &g_gc_card_table[card_word (gcard_of (la))];
7344 //clear the card table
7346 memclr ((uint8_t*)ct,
7347 (((saved_g_highest_address - saved_g_lowest_address)*sizeof (uint32_t) /
7348 (card_size * card_word_width))
7349 + sizeof (uint32_t)));
7352 bt = (short*)((uint8_t*)ct + cs);
7354 // No initialization needed, will be done in copy_brick_card
7356 card_table_brick_table (ct) = bt;
7359 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7360 //set all bundle to look at all of the cards
7361 memset(card_table_card_bundle_table (ct), 0xFF, cb);
7362 #endif //CARD_BUNDLE
7364 #ifdef GROWABLE_SEG_MAPPING_TABLE
7366 new_seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7367 new_seg_mapping_table = (seg_mapping*)((uint8_t*)new_seg_mapping_table -
7368 size_seg_mapping_table_of (0, (align_lower_segment (saved_g_lowest_address))));
7369 memcpy(&new_seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7370 &seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7371 size_seg_mapping_table_of(g_gc_lowest_address, g_gc_highest_address));
7373 // new_seg_mapping_table gets assigned to seg_mapping_table at the bottom of this function,
7374 // not here. The reason for this is that, if we fail at mark array committing (OOM) and we've
7375 // already switched seg_mapping_table to point to the new mapping table, we'll decommit it and
7376 // run into trouble. By not assigning here, we're making sure that we will not change seg_mapping_table
7377 // if an OOM occurs.
7379 #endif //GROWABLE_SEG_MAPPING_TABLE
7382 if(gc_can_use_concurrent)
7383 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7385 card_table_mark_array (ct) = NULL;
7388 translated_ct = translate_card_table (ct);
7390 dprintf (GC_TABLE_LOG, ("card table: %Ix(translated: %Ix), seg map: %Ix, mark array: %Ix",
7391 (size_t)ct, (size_t)translated_ct, (size_t)new_seg_mapping_table, (size_t)card_table_mark_array (ct)));
7393 #ifdef BACKGROUND_GC
7394 if (hp->should_commit_mark_array())
7396 dprintf (GC_TABLE_LOG, ("new low: %Ix, new high: %Ix, latest mark array is %Ix(translate: %Ix)",
7397 saved_g_lowest_address, saved_g_highest_address,
7398 card_table_mark_array (ct),
7399 translate_mark_array (card_table_mark_array (ct))));
7400 uint32_t* new_mark_array = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, saved_g_lowest_address));
7401 if (!commit_new_mark_array_global (new_mark_array))
7403 dprintf (GC_TABLE_LOG, ("failed to commit portions in the mark array for existing segments"));
7404 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7408 if (!commit_mark_array_new_seg (hp, new_seg, translated_ct, saved_g_lowest_address))
7410 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg"));
7411 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7417 clear_commit_flag_global();
7419 #endif //BACKGROUND_GC
7421 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7422 if (gc_can_use_concurrent)
7424 // The current design of software write watch requires that the runtime is suspended during resize. Suspending
7425 // on resize is preferred because it is a far less frequent operation than GetWriteWatch() / ResetWriteWatch().
7426 // Suspending here allows copying dirty state from the old table into the new table, and not have to merge old
7427 // table info lazily as done for card tables.
7429 // Either this thread was the thread that did the suspension which means we are suspended; or this is called
7430 // from a GC thread which means we are in a blocking GC and also suspended.
7431 bool is_runtime_suspended = GCToEEInterface::IsGCThread();
7432 if (!is_runtime_suspended)
7434 // Note on points where the runtime is suspended anywhere in this function. Upon an attempt to suspend the
7435 // runtime, a different thread may suspend first, causing this thread to block at the point of the suspend call.
7436 // So, at any suspend point, externally visible state needs to be consistent, as code that depends on that state
7437 // may run while this thread is blocked. This includes updates to g_gc_card_table, g_gc_lowest_address, and
7438 // g_gc_highest_address.
7442 g_gc_card_table = translated_ct;
7444 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7445 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7448 SoftwareWriteWatch::SetResizedUntranslatedTable(
7449 mem + sw_ww_table_offset,
7450 saved_g_lowest_address,
7451 saved_g_highest_address);
7453 // Since the runtime is already suspended, update the write barrier here as well.
7454 // This passes a bool telling whether we need to switch to the post
7455 // grow version of the write barrier. This test tells us if the new
7456 // segment was allocated at a lower address than the old, requiring
7457 // that we start doing an upper bounds check in the write barrier.
7458 g_gc_lowest_address = saved_g_lowest_address;
7459 g_gc_highest_address = saved_g_highest_address;
7460 stomp_write_barrier_resize(true, la != saved_g_lowest_address);
7461 write_barrier_updated = true;
7463 if (!is_runtime_suspended)
7469 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7471 g_gc_card_table = translated_ct;
7473 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7474 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7478 seg_mapping_table = new_seg_mapping_table;
7480 GCToOSInterface::FlushProcessWriteBuffers();
7481 g_gc_lowest_address = saved_g_lowest_address;
7482 g_gc_highest_address = saved_g_highest_address;
7484 if (!write_barrier_updated)
7486 // This passes a bool telling whether we need to switch to the post
7487 // grow version of the write barrier. This test tells us if the new
7488 // segment was allocated at a lower address than the old, requiring
7489 // that we start doing an upper bounds check in the write barrier.
7490 // This will also suspend the runtime if the write barrier type needs
7491 // to be changed, so we are doing this after all global state has
7492 // been updated. See the comment above suspend_EE() above for more
7494 stomp_write_barrier_resize(GCToEEInterface::IsGCThread(), la != saved_g_lowest_address);
7501 //cleanup mess and return -1;
7505 assert(g_gc_card_table == saved_g_card_table);
7507 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7508 assert(g_gc_card_bundle_table == saved_g_card_bundle_table);
7511 //delete (uint32_t*)((uint8_t*)ct - sizeof(card_table_info));
7512 if (!GCToOSInterface::VirtualRelease (mem, alloc_size))
7514 dprintf (GC_TABLE_LOG, ("GCToOSInterface::VirtualRelease failed"));
7515 assert (!"release failed");
7523 #ifdef BACKGROUND_GC
7524 if (hp->should_commit_mark_array())
7526 dprintf (GC_TABLE_LOG, ("in range new seg %Ix, mark_array is %Ix", new_seg, hp->mark_array));
7527 if (!commit_mark_array_new_seg (hp, new_seg))
7529 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg in range"));
7530 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7534 #endif //BACKGROUND_GC
7540 //copy all of the arrays managed by the card table for a page aligned range
7541 void gc_heap::copy_brick_card_range (uint8_t* la, uint32_t* old_card_table,
7542 short* old_brick_table,
7544 uint8_t* start, uint8_t* end)
7546 ptrdiff_t brick_offset = brick_of (start) - brick_of (la);
7549 dprintf (2, ("copying tables for range [%Ix %Ix[", (size_t)start, (size_t)end));
7552 short* brick_start = &brick_table [brick_of (start)];
7553 if (old_brick_table)
7555 // segments are always on page boundaries
7556 memcpy (brick_start, &old_brick_table[brick_offset],
7557 size_brick_of (start, end));
7562 // This is a large heap, just clear the brick table
7565 uint32_t* old_ct = &old_card_table[card_word (card_of (la))];
7567 #ifdef BACKGROUND_GC
7568 UNREFERENCED_PARAMETER(seg);
7569 if (recursive_gc_sync::background_running_p())
7571 uint32_t* old_mark_array = card_table_mark_array (old_ct);
7573 // We don't need to go through all the card tables here because
7574 // we only need to copy from the GC version of the mark array - when we
7575 // mark (even in allocate_large_object) we always use that mark array.
7576 if ((card_table_highest_address (old_ct) >= start) &&
7577 (card_table_lowest_address (old_ct) <= end))
7579 if ((background_saved_highest_address >= start) &&
7580 (background_saved_lowest_address <= end))
7582 //copy the mark bits
7583 // segments are always on page boundaries
7584 uint8_t* m_start = max (background_saved_lowest_address, start);
7585 uint8_t* m_end = min (background_saved_highest_address, end);
7586 memcpy (&mark_array[mark_word_of (m_start)],
7587 &old_mark_array[mark_word_of (m_start) - mark_word_of (la)],
7588 size_mark_array_of (m_start, m_end));
7593 //only large segments can be out of range
7594 assert (old_brick_table == 0);
7597 #else //BACKGROUND_GC
7599 clear_mark_array (start, heap_segment_committed(seg));
7600 #endif //BACKGROUND_GC
7603 // n way merge with all of the card table ever used in between
7604 uint32_t* ct = card_table_next (&card_table[card_word (card_of(lowest_address))]);
7607 while (card_table_next (old_ct) != ct)
7609 //copy if old card table contained [start, end[
7610 if ((card_table_highest_address (ct) >= end) &&
7611 (card_table_lowest_address (ct) <= start))
7613 // or the card_tables
7615 size_t start_word = card_word (card_of (start));
7617 uint32_t* dest = &card_table[start_word];
7618 uint32_t* src = &((translate_card_table (ct))[start_word]);
7619 ptrdiff_t count = count_card_of (start, end);
7620 for (int x = 0; x < count; x++)
7624 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7627 card_bundle_set(cardw_card_bundle(start_word+x));
7635 ct = card_table_next (ct);
7639 //initialize all of the arrays managed by the card table for a page aligned range when an existing ro segment becomes in range
7640 void gc_heap::init_brick_card_range (heap_segment* seg)
7642 dprintf (2, ("initialising tables for range [%Ix %Ix[",
7643 (size_t)heap_segment_mem (seg),
7644 (size_t)heap_segment_allocated (seg)));
7646 // initialize the brick table
7647 for (size_t b = brick_of (heap_segment_mem (seg));
7648 b < brick_of (align_on_brick (heap_segment_allocated (seg)));
7655 if (recursive_gc_sync::background_running_p() && (seg->flags & heap_segment_flags_ma_committed))
7658 clear_mark_array (heap_segment_mem (seg), heap_segment_committed(seg));
7662 clear_card_for_addresses (heap_segment_mem (seg),
7663 heap_segment_allocated (seg));
7666 void gc_heap::copy_brick_card_table()
7668 uint8_t* la = lowest_address;
7669 uint8_t* ha = highest_address;
7670 MAYBE_UNUSED_VAR(ha);
7671 uint32_t* old_card_table = card_table;
7672 short* old_brick_table = brick_table;
7674 assert (la == card_table_lowest_address (&old_card_table[card_word (card_of (la))]));
7675 assert (ha == card_table_highest_address (&old_card_table[card_word (card_of (la))]));
7677 /* todo: Need a global lock for this */
7678 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
7679 own_card_table (ct);
7680 card_table = translate_card_table (ct);
7681 /* End of global lock */
7682 highest_address = card_table_highest_address (ct);
7683 lowest_address = card_table_lowest_address (ct);
7685 brick_table = card_table_brick_table (ct);
7688 if (gc_can_use_concurrent)
7690 mark_array = translate_mark_array (card_table_mark_array (ct));
7691 assert (mark_word_of (g_gc_highest_address) ==
7692 mark_word_of (align_on_mark_word (g_gc_highest_address)));
7699 #if defined(MARK_ARRAY) && defined(_DEBUG)
7700 #ifdef GROWABLE_SEG_MAPPING_TABLE
7701 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7702 #else //GROWABLE_SEG_MAPPING_TABLE
7704 #endif //GROWABLE_SEG_MAPPING_TABLE
7705 #endif //MARK_ARRAY && _DEBUG
7706 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
7708 // Ensure that the word that represents g_gc_lowest_address in the translated table is located at the
7709 // start of the untranslated table.
7710 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
7711 card_table_card_bundle_table (ct));
7713 //set the card table if we are in a heap growth scenario
7714 if (card_bundles_enabled())
7716 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
7717 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
7719 //check if we need to turn on card_bundles.
7720 #ifdef MULTIPLE_HEAPS
7721 // use INT64 arithmetic here because of possible overflow on 32p
7722 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*gc_heap::n_heaps;
7724 // use INT64 arithmetic here because of possible overflow on 32p
7725 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
7726 #endif //MULTIPLE_HEAPS
7727 if (reserved_memory >= th)
7729 enable_card_bundles();
7732 #endif //CARD_BUNDLE
7734 // for each of the segments and heaps, copy the brick table and
7735 // or the card table
7736 heap_segment* seg = generation_start_segment (generation_of (max_generation));
7739 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7741 //check if it became in range
7742 if ((heap_segment_reserved (seg) > lowest_address) &&
7743 (heap_segment_mem (seg) < highest_address))
7745 set_ro_segment_in_range (seg);
7751 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7752 copy_brick_card_range (la, old_card_table,
7755 align_lower_page (heap_segment_mem (seg)),
7758 seg = heap_segment_next (seg);
7761 seg = generation_start_segment (large_object_generation);
7764 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7766 //check if it became in range
7767 if ((heap_segment_reserved (seg) > lowest_address) &&
7768 (heap_segment_mem (seg) < highest_address))
7770 set_ro_segment_in_range (seg);
7775 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7776 copy_brick_card_range (la, old_card_table,
7779 align_lower_page (heap_segment_mem (seg)),
7782 seg = heap_segment_next (seg);
7785 release_card_table (&old_card_table[card_word (card_of(la))]);
7788 #ifdef FEATURE_BASICFREEZE
7789 BOOL gc_heap::insert_ro_segment (heap_segment* seg)
7791 enter_spin_lock (&gc_heap::gc_lock);
7793 if (!gc_heap::seg_table->ensure_space_for_insert ()
7794 || (should_commit_mark_array() && !commit_mark_array_new_seg(__this, seg)))
7796 leave_spin_lock(&gc_heap::gc_lock);
7800 //insert at the head of the segment list
7801 generation* gen2 = generation_of (max_generation);
7802 heap_segment* oldhead = generation_start_segment (gen2);
7803 heap_segment_next (seg) = oldhead;
7804 generation_start_segment (gen2) = seg;
7806 seg_table->insert (heap_segment_mem(seg), (size_t)seg);
7808 #ifdef SEG_MAPPING_TABLE
7809 seg_mapping_table_add_ro_segment (seg);
7810 #endif //SEG_MAPPING_TABLE
7813 if ((heap_segment_reserved (seg) > lowest_address) &&
7814 (heap_segment_mem (seg) < highest_address))
7816 set_ro_segment_in_range (seg);
7819 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg), (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)), gc_etw_segment_read_only_heap);
7821 leave_spin_lock (&gc_heap::gc_lock);
7825 // No one is calling this function right now. If this is getting called we need
7826 // to take care of decommitting the mark array for it - we will need to remember
7827 // which portion of the mark array was committed and only decommit that.
7828 void gc_heap::remove_ro_segment (heap_segment* seg)
7830 //clear the mark bits so a new segment allocated in its place will have a clear mark bits
7832 if (gc_can_use_concurrent)
7834 clear_mark_array (align_lower_mark_word (max (heap_segment_mem (seg), lowest_address)),
7835 align_on_card_word (min (heap_segment_allocated (seg), highest_address)),
7836 false); // read_only segments need the mark clear
7840 enter_spin_lock (&gc_heap::gc_lock);
7842 seg_table->remove ((uint8_t*)seg);
7844 #ifdef SEG_MAPPING_TABLE
7845 seg_mapping_table_remove_ro_segment (seg);
7846 #endif //SEG_MAPPING_TABLE
7848 // Locate segment (and previous segment) in the list.
7849 generation* gen2 = generation_of (max_generation);
7850 heap_segment* curr_seg = generation_start_segment (gen2);
7851 heap_segment* prev_seg = NULL;
7853 while (curr_seg && curr_seg != seg)
7855 prev_seg = curr_seg;
7856 curr_seg = heap_segment_next (curr_seg);
7858 assert (curr_seg == seg);
7860 // Patch previous segment (or list head if there is none) to skip the removed segment.
7862 heap_segment_next (prev_seg) = heap_segment_next (curr_seg);
7864 generation_start_segment (gen2) = heap_segment_next (curr_seg);
7866 leave_spin_lock (&gc_heap::gc_lock);
7868 #endif //FEATURE_BASICFREEZE
7870 BOOL gc_heap::set_ro_segment_in_range (heap_segment* seg)
7873 seg->flags |= heap_segment_flags_inrange;
7874 // init_brick_card_range (seg);
7875 ro_segments_in_range = TRUE;
7876 //right now, segments aren't protected
7877 //unprotect_segment (seg);
7883 uint8_t** make_mark_list (size_t size)
7885 uint8_t** mark_list = new (nothrow) uint8_t* [size];
7889 #define swap(a,b){uint8_t* t; t = a; a = b; b = t;}
7891 void verify_qsort_array (uint8_t* *low, uint8_t* *high)
7895 for (i = low+1; i <= high; i++)
7904 #ifndef USE_INTROSORT
7905 void qsort1( uint8_t* *low, uint8_t* *high, unsigned int depth)
7907 if (((low + 16) >= high) || (depth > 100))
7911 for (i = low+1; i <= high; i++)
7914 for (j=i;j >low && val<*(j-1);j--)
7923 uint8_t *pivot, **left, **right;
7925 //sort low middle and high
7926 if (*(low+((high-low)/2)) < *low)
7927 swap (*(low+((high-low)/2)), *low);
7930 if (*high < *(low+((high-low)/2)))
7931 swap (*(low+((high-low)/2)), *high);
7933 swap (*(low+((high-low)/2)), *(high-1));
7935 left = low; right = high-1;
7937 while (*(--right) > pivot);
7938 while (*(++left) < pivot);
7941 swap(*left, *right);
7946 swap (*left, *(high-1));
7947 qsort1(low, left-1, depth+1);
7948 qsort1(left+1, high, depth+1);
7951 #endif //USE_INTROSORT
7952 void rqsort1( uint8_t* *low, uint8_t* *high)
7954 if ((low + 16) >= high)
7958 for (i = low+1; i <= high; i++)
7961 for (j=i;j >low && val>*(j-1);j--)
7970 uint8_t *pivot, **left, **right;
7972 //sort low middle and high
7973 if (*(low+((high-low)/2)) > *low)
7974 swap (*(low+((high-low)/2)), *low);
7977 if (*high > *(low+((high-low)/2)))
7978 swap (*(low+((high-low)/2)), *high);
7980 swap (*(low+((high-low)/2)), *(high-1));
7982 left = low; right = high-1;
7984 while (*(--right) < pivot);
7985 while (*(++left) > pivot);
7988 swap(*left, *right);
7993 swap (*left, *(high-1));
7994 rqsort1(low, left-1);
7995 rqsort1(left+1, high);
7999 #ifdef USE_INTROSORT
8004 static const int size_threshold = 64;
8005 static const int max_depth = 100;
8008 inline static void swap_elements(uint8_t** i,uint8_t** j)
8016 static void sort (uint8_t** begin, uint8_t** end, int ignored)
8019 introsort_loop (begin, end, max_depth);
8020 insertionsort (begin, end);
8025 static void introsort_loop (uint8_t** lo, uint8_t** hi, int depth_limit)
8027 while (hi-lo >= size_threshold)
8029 if (depth_limit == 0)
8034 uint8_t** p=median_partition (lo, hi);
8035 depth_limit=depth_limit-1;
8036 introsort_loop (p, hi, depth_limit);
8041 static uint8_t** median_partition (uint8_t** low, uint8_t** high)
8043 uint8_t *pivot, **left, **right;
8045 //sort low middle and high
8046 if (*(low+((high-low)/2)) < *low)
8047 swap_elements ((low+((high-low)/2)), low);
8049 swap_elements (low, high);
8050 if (*high < *(low+((high-low)/2)))
8051 swap_elements ((low+((high-low)/2)), high);
8053 swap_elements ((low+((high-low)/2)), (high-1));
8055 left = low; right = high-1;
8057 while (*(--right) > pivot);
8058 while (*(++left) < pivot);
8061 swap_elements(left, right);
8066 swap_elements (left, (high-1));
8071 static void insertionsort (uint8_t** lo, uint8_t** hi)
8073 for (uint8_t** i=lo+1; i <= hi; i++)
8077 while((j > lo) && (t <*(j-1)))
8086 static void heapsort (uint8_t** lo, uint8_t** hi)
8088 size_t n = hi - lo + 1;
8089 for (size_t i=n / 2; i >= 1; i--)
8093 for (size_t i = n; i > 1; i--)
8095 swap_elements (lo, lo + i - 1);
8096 downheap(1, i - 1, lo);
8100 static void downheap (size_t i, size_t n, uint8_t** lo)
8102 uint8_t* d = *(lo + i - 1);
8107 if (child < n && *(lo + child - 1)<(*(lo + child)))
8111 if (!(d<*(lo + child - 1)))
8115 *(lo + i - 1) = *(lo + child - 1);
8123 #endif //USE_INTROSORT
8125 #ifdef MULTIPLE_HEAPS
8126 #ifdef PARALLEL_MARK_LIST_SORT
8127 void gc_heap::sort_mark_list()
8129 // if this heap had a mark list overflow, we don't do anything
8130 if (mark_list_index > mark_list_end)
8132 // printf("sort_mark_list: overflow on heap %d\n", heap_number);
8136 // if any other heap had a mark list overflow, we fake one too,
8137 // so we don't use an incomplete mark list by mistake
8138 for (int i = 0; i < n_heaps; i++)
8140 if (g_heaps[i]->mark_list_index > g_heaps[i]->mark_list_end)
8142 mark_list_index = mark_list_end + 1;
8143 // printf("sort_mark_list: overflow on heap %d\n", i);
8148 // unsigned long start = GetCycleCount32();
8150 dprintf (3, ("Sorting mark lists"));
8151 if (mark_list_index > mark_list)
8152 _sort (mark_list, mark_list_index - 1, 0);
8154 // printf("first phase of sort_mark_list for heap %d took %u cycles to sort %u entries\n", this->heap_number, GetCycleCount32() - start, mark_list_index - mark_list);
8155 // start = GetCycleCount32();
8157 // first set the pieces for all heaps to empty
8159 for (heap_num = 0; heap_num < n_heaps; heap_num++)
8161 mark_list_piece_start[heap_num] = NULL;
8162 mark_list_piece_end[heap_num] = NULL;
8165 uint8_t** x = mark_list;
8167 // predicate means: x is still within the mark list, and within the bounds of this heap
8168 #define predicate(x) (((x) < mark_list_index) && (*(x) < heap->ephemeral_high))
8171 while (x < mark_list_index)
8174 // find the heap x points into - searching cyclically from the last heap,
8175 // because in many cases the right heap is the next one or comes soon after
8176 int last_heap_num = heap_num;
8177 MAYBE_UNUSED_VAR(last_heap_num);
8181 if (heap_num >= n_heaps)
8183 assert(heap_num != last_heap_num); // we should always find the heap - infinite loop if not!
8184 heap = g_heaps[heap_num];
8186 while (!(*x >= heap->ephemeral_low && *x < heap->ephemeral_high));
8188 // x is the start of the mark list piece for this heap
8189 mark_list_piece_start[heap_num] = x;
8191 // to find the end of the mark list piece for this heap, find the first x
8192 // that has !predicate(x), i.e. that is either not in this heap, or beyond the end of the list
8195 // let's see if we get lucky and the whole rest belongs to this piece
8196 if (predicate(mark_list_index-1))
8198 x = mark_list_index;
8199 mark_list_piece_end[heap_num] = x;
8203 // we play a variant of binary search to find the point sooner.
8204 // the first loop advances by increasing steps until the predicate turns false.
8205 // then we retreat the last step, and the second loop advances by decreasing steps, keeping the predicate true.
8210 uint8_t** temp_x = x;
8217 while (predicate(x));
8218 // we know that only the last step was wrong, so we undo it
8222 // loop invariant - predicate holds at x, but not x + inc
8223 assert (predicate(x) && !(((x + inc) > x) && predicate(x + inc)));
8225 if (((x + inc) > x) && predicate(x + inc))
8231 // the termination condition and the loop invariant together imply this:
8232 assert(predicate(x) && !predicate(x + inc) && (inc == 1));
8233 // so the spot we're looking for is one further
8236 mark_list_piece_end[heap_num] = x;
8241 // printf("second phase of sort_mark_list for heap %d took %u cycles\n", this->heap_number, GetCycleCount32() - start);
8244 void gc_heap::append_to_mark_list(uint8_t **start, uint8_t **end)
8246 size_t slots_needed = end - start;
8247 size_t slots_available = mark_list_end + 1 - mark_list_index;
8248 size_t slots_to_copy = min(slots_needed, slots_available);
8249 memcpy(mark_list_index, start, slots_to_copy*sizeof(*start));
8250 mark_list_index += slots_to_copy;
8251 // printf("heap %d: appended %Id slots to mark_list\n", heap_number, slots_to_copy);
8254 void gc_heap::merge_mark_lists()
8256 uint8_t** source[MAX_SUPPORTED_CPUS];
8257 uint8_t** source_end[MAX_SUPPORTED_CPUS];
8258 int source_heap[MAX_SUPPORTED_CPUS];
8259 int source_count = 0;
8261 // in case of mark list overflow, don't bother
8262 if (mark_list_index > mark_list_end)
8264 // printf("merge_mark_lists: overflow\n");
8268 dprintf(3, ("merge_mark_lists: heap_number = %d starts out with %Id entries", heap_number, mark_list_index - mark_list));
8269 // unsigned long start = GetCycleCount32();
8270 for (int i = 0; i < n_heaps; i++)
8272 gc_heap* heap = g_heaps[i];
8273 if (heap->mark_list_piece_start[heap_number] < heap->mark_list_piece_end[heap_number])
8275 source[source_count] = heap->mark_list_piece_start[heap_number];
8276 source_end[source_count] = heap->mark_list_piece_end[heap_number];
8277 source_heap[source_count] = i;
8278 if (source_count < MAX_SUPPORTED_CPUS)
8282 // printf("first phase of merge_mark_lists for heap %d took %u cycles\n", heap_number, GetCycleCount32() - start);
8284 dprintf(3, ("heap_number = %d has %d sources\n", heap_number, source_count));
8285 #if defined(_DEBUG) || defined(TRACE_GC)
8286 for (int j = 0; j < source_count; j++)
8288 dprintf(3, ("heap_number = %d ", heap_number));
8289 dprintf(3, (" source from heap %d = %Ix .. %Ix (%Id entries)",
8290 (size_t)(source_heap[j]), (size_t)(source[j][0]), (size_t)(source_end[j][-1]), (size_t)(source_end[j] - source[j])));
8291 // the sources should all be sorted
8292 for (uint8_t **x = source[j]; x < source_end[j] - 1; x++)
8296 dprintf(3, ("oops, mark_list from source %d for heap %d isn't sorted\n", j, heap_number));
8301 #endif //_DEBUG || TRACE_GC
8303 // start = GetCycleCount32();
8305 mark_list = &g_mark_list_copy [heap_number*mark_list_size];
8306 mark_list_index = mark_list;
8307 mark_list_end = &mark_list [mark_list_size-1];
8308 int piece_count = 0;
8309 if (source_count == 0)
8313 else if (source_count == 1)
8315 mark_list = source[0];
8316 mark_list_index = source_end[0];
8317 mark_list_end = mark_list_index;
8322 while (source_count > 1)
8324 // find the lowest and second lowest value in the sources we're merging from
8325 int lowest_source = 0;
8326 uint8_t *lowest = *source[0];
8327 uint8_t *second_lowest = *source[1];
8328 for (int i = 1; i < source_count; i++)
8330 if (lowest > *source[i])
8332 second_lowest = lowest;
8333 lowest = *source[i];
8336 else if (second_lowest > *source[i])
8338 second_lowest = *source[i];
8342 // find the point in the lowest source where it either runs out or is not <= second_lowest anymore
8344 // let's first try to get lucky and see if the whole source is <= second_lowest -- this is actually quite common
8346 if (source_end[lowest_source][-1] <= second_lowest)
8347 x = source_end[lowest_source];
8350 // use linear search to find the end -- could also use binary search as in sort_mark_list,
8351 // but saw no improvement doing that
8352 for (x = source[lowest_source]; x < source_end[lowest_source] && *x <= second_lowest; x++)
8356 // blast this piece to the mark list
8357 append_to_mark_list(source[lowest_source], x);
8360 source[lowest_source] = x;
8362 // check whether this source is now exhausted
8363 if (x >= source_end[lowest_source])
8365 // if it's not the source with the highest index, copy the source with the highest index
8366 // over it so the non-empty sources are always at the beginning
8367 if (lowest_source < source_count-1)
8369 source[lowest_source] = source[source_count-1];
8370 source_end[lowest_source] = source_end[source_count-1];
8375 // we're left with just one source that we copy
8376 append_to_mark_list(source[0], source_end[0]);
8380 // printf("second phase of merge_mark_lists for heap %d took %u cycles to merge %d pieces\n", heap_number, GetCycleCount32() - start, piece_count);
8382 #if defined(_DEBUG) || defined(TRACE_GC)
8383 // the final mark list must be sorted
8384 for (uint8_t **x = mark_list; x < mark_list_index - 1; x++)
8388 dprintf(3, ("oops, mark_list for heap %d isn't sorted at the end of merge_mark_lists", heap_number));
8392 #endif //defined(_DEBUG) || defined(TRACE_GC)
8394 #else //PARALLEL_MARK_LIST_SORT
8395 void gc_heap::combine_mark_lists()
8397 dprintf (3, ("Combining mark lists"));
8398 //verify if a heap has overflowed its mark list
8399 BOOL use_mark_list = TRUE;
8400 for (int i = 0; i < n_heaps; i++)
8402 if (g_heaps [i]->mark_list_index > g_heaps [i]->mark_list_end)
8404 use_mark_list = FALSE;
8411 dprintf (3, ("Using mark list"));
8412 //compact the gaps out of the mark list
8414 uint8_t** current_gap = g_heaps [gn]->mark_list_index;
8415 uint8_t** current_gap_end = g_heaps[gn]->mark_list_end + 1;
8416 uint8_t** dst_last = current_gap-1;
8418 int srcn = n_heaps-1;
8419 gc_heap* srch = g_heaps [srcn];
8420 uint8_t** src = srch->mark_list_index - 1;
8421 uint8_t** src_beg = srch->mark_list;
8423 while (current_gap <= src)
8425 while ((gn < n_heaps-1) && (current_gap >= current_gap_end))
8427 //go to the next gap
8429 dprintf (3, ("Going to the next gap %d", gn));
8430 assert (gn < n_heaps);
8431 current_gap = g_heaps [gn]->mark_list_index;
8432 current_gap_end = g_heaps[gn]->mark_list_end + 1;
8433 assert ((gn == (n_heaps-1)) || (current_gap_end == g_heaps[gn+1]->mark_list));
8435 while ((srcn > 0) && (src < src_beg))
8437 //go to the previous source
8439 dprintf (3, ("going to the previous source %d", srcn));
8441 gc_heap* srch = g_heaps [srcn];
8442 src = srch->mark_list_index - 1;
8443 src_beg = srch->mark_list;
8445 if (current_gap < src)
8447 dst_last = current_gap;
8448 *current_gap++ = *src--;
8451 dprintf (3, ("src: %Ix dst_last: %Ix", (size_t)src, (size_t)dst_last));
8453 uint8_t** end_of_list = max (src, dst_last);
8455 //sort the resulting compacted list
8456 assert (end_of_list < &g_mark_list [n_heaps*mark_list_size]);
8457 if (end_of_list > &g_mark_list[0])
8458 _sort (&g_mark_list[0], end_of_list, 0);
8459 //adjust the mark_list to the begining of the resulting mark list.
8460 for (int i = 0; i < n_heaps; i++)
8462 g_heaps [i]->mark_list = g_mark_list;
8463 g_heaps [i]->mark_list_index = end_of_list + 1;
8464 g_heaps [i]->mark_list_end = end_of_list + 1;
8469 uint8_t** end_of_list = g_mark_list;
8470 //adjust the mark_list to the begining of the resulting mark list.
8471 //put the index beyond the end to turn off mark list processing
8472 for (int i = 0; i < n_heaps; i++)
8474 g_heaps [i]->mark_list = g_mark_list;
8475 g_heaps [i]->mark_list_index = end_of_list + 1;
8476 g_heaps [i]->mark_list_end = end_of_list;
8480 #endif // PARALLEL_MARK_LIST_SORT
8481 #endif //MULTIPLE_HEAPS
8484 class seg_free_spaces
8486 struct seg_free_space
8492 struct free_space_bucket
8494 seg_free_space* free_space;
8495 ptrdiff_t count_add; // Assigned when we first contruct the array.
8496 ptrdiff_t count_fit; // How many items left when we are fitting plugs.
8499 void move_bucket (int old_power2, int new_power2)
8501 // PREFAST warning 22015: old_power2 could be negative
8502 assert (old_power2 >= 0);
8503 assert (old_power2 >= new_power2);
8505 if (old_power2 == new_power2)
8510 seg_free_space* src_index = free_space_buckets[old_power2].free_space;
8511 for (int i = old_power2; i > new_power2; i--)
8513 seg_free_space** dest = &(free_space_buckets[i].free_space);
8516 seg_free_space* dest_index = free_space_buckets[i - 1].free_space;
8517 if (i > (new_power2 + 1))
8519 seg_free_space temp = *src_index;
8520 *src_index = *dest_index;
8523 src_index = dest_index;
8526 free_space_buckets[old_power2].count_fit--;
8527 free_space_buckets[new_power2].count_fit++;
8532 void dump_free_space (seg_free_space* item)
8539 mark* m = (mark*)(item->start);
8540 len = pinned_len (m);
8541 addr = pinned_plug (m) - len;
8545 heap_segment* seg = (heap_segment*)(item->start);
8546 addr = heap_segment_plan_allocated (seg);
8547 len = heap_segment_committed (seg) - addr;
8550 dprintf (SEG_REUSE_LOG_1, ("[%d]0x%Ix %Id", heap_num, addr, len));
8555 seg_free_space* item = NULL;
8558 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------\nnow the free spaces look like:", heap_num));
8559 for (i = 0; i < (free_space_bucket_count - 1); i++)
8561 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8562 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8563 item = free_space_buckets[i].free_space;
8564 while (item < free_space_buckets[i + 1].free_space)
8566 dump_free_space (item);
8569 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8572 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8573 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8574 item = free_space_buckets[i].free_space;
8576 while (item <= &seg_free_space_array[free_space_item_count - 1])
8578 dump_free_space (item);
8581 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8586 free_space_bucket* free_space_buckets;
8587 seg_free_space* seg_free_space_array;
8588 ptrdiff_t free_space_bucket_count;
8589 ptrdiff_t free_space_item_count;
8593 BOOL has_end_of_seg;
8598 seg_free_spaces (int h_number)
8600 heap_num = h_number;
8605 size_t total_prealloc_size =
8606 MAX_NUM_BUCKETS * sizeof (free_space_bucket) +
8607 MAX_NUM_FREE_SPACES * sizeof (seg_free_space);
8609 free_space_buckets = (free_space_bucket*) new (nothrow) uint8_t[total_prealloc_size];
8611 return (!!free_space_buckets);
8614 // We take the ordered free space array we got from the 1st pass,
8615 // and feed the portion that we decided to use to this method, ie,
8616 // the largest item_count free spaces.
8617 void add_buckets (int base, size_t* ordered_free_spaces, int bucket_count, size_t item_count)
8619 assert (free_space_buckets);
8620 assert (item_count <= (size_t)MAX_PTR);
8622 free_space_bucket_count = bucket_count;
8623 free_space_item_count = item_count;
8626 has_end_of_seg = FALSE;
8629 ptrdiff_t total_item_count = 0;
8632 seg_free_space_array = (seg_free_space*)(free_space_buckets + free_space_bucket_count);
8634 for (i = 0; i < (ptrdiff_t)item_count; i++)
8636 seg_free_space_array[i].start = 0;
8637 seg_free_space_array[i].is_plug = FALSE;
8640 for (i = 0; i < bucket_count; i++)
8642 free_space_buckets[i].count_add = ordered_free_spaces[i];
8643 free_space_buckets[i].count_fit = ordered_free_spaces[i];
8644 free_space_buckets[i].free_space = &seg_free_space_array[total_item_count];
8645 total_item_count += free_space_buckets[i].count_add;
8648 assert (total_item_count == (ptrdiff_t)item_count);
8651 // If we are adding a free space before a plug we pass the
8652 // mark stack position so we can update the length; we could
8653 // also be adding the free space after the last plug in which
8654 // case start is the segment which we'll need to update the
8655 // heap_segment_plan_allocated.
8656 void add (void* start, BOOL plug_p, BOOL first_p)
8658 size_t size = (plug_p ?
8659 pinned_len ((mark*)start) :
8660 (heap_segment_committed ((heap_segment*)start) -
8661 heap_segment_plan_allocated ((heap_segment*)start)));
8665 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space before plug: %Id", heap_num, size));
8669 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space at end of seg: %Id", heap_num, size));
8671 has_end_of_seg = TRUE;
8677 size_t eph_gen_starts = gc_heap::eph_gen_starts_size;
8678 size -= eph_gen_starts;
8681 mark* m = (mark*)(start);
8682 pinned_len (m) -= eph_gen_starts;
8686 heap_segment* seg = (heap_segment*)start;
8687 heap_segment_plan_allocated (seg) += eph_gen_starts;
8691 int bucket_power2 = index_of_set_bit (round_down_power2 (size));
8692 if (bucket_power2 < base_power2)
8697 free_space_bucket* bucket = &free_space_buckets[bucket_power2 - base_power2];
8699 seg_free_space* bucket_free_space = bucket->free_space;
8700 assert (plug_p || (!plug_p && bucket->count_add));
8702 if (bucket->count_add == 0)
8704 dprintf (SEG_REUSE_LOG_1, ("[%d]Already have enough of 2^%d", heap_num, bucket_power2));
8708 ptrdiff_t index = bucket->count_add - 1;
8710 dprintf (SEG_REUSE_LOG_1, ("[%d]Building free spaces: adding %Ix; len: %Id (2^%d)",
8713 (pinned_plug ((mark*)start) - pinned_len ((mark*)start)) :
8714 heap_segment_plan_allocated ((heap_segment*)start)),
8720 bucket_free_space[index].is_plug = TRUE;
8723 bucket_free_space[index].start = start;
8724 bucket->count_add--;
8729 // Do a consistency check after all free spaces are added.
8733 int end_of_seg_count = 0;
8735 for (i = 0; i < free_space_item_count; i++)
8737 assert (seg_free_space_array[i].start);
8738 if (!(seg_free_space_array[i].is_plug))
8746 assert (end_of_seg_count == 1);
8750 assert (end_of_seg_count == 0);
8753 for (i = 0; i < free_space_bucket_count; i++)
8755 assert (free_space_buckets[i].count_add == 0);
8761 uint8_t* fit (uint8_t* old_loc,
8763 BOOL set_padding_on_saved_p,
8764 mark* pinned_plug_entry,
8765 #endif //SHORT_PLUGS
8767 REQD_ALIGN_AND_OFFSET_DCL)
8772 assert (!is_plug_padded (old_loc));
8773 #endif //SHORT_PLUGS
8774 assert (!node_realigned (old_loc));
8777 size_t saved_plug_size = plug_size;
8779 #ifdef FEATURE_STRUCTALIGN
8780 // BARTOKTODO (4841): this code path is disabled (see can_fit_all_blocks_p) until we take alignment requirements into account
8781 _ASSERTE(requiredAlignment == DATA_ALIGNMENT && false);
8782 #endif // FEATURE_STRUCTALIGN
8783 // TODO: this is also not large alignment ready. We would need to consider alignment when chosing the
8786 size_t plug_size_to_fit = plug_size;
8788 int pad_in_front = (old_loc != 0) ? USE_PADDING_FRONT : 0;
8791 plug_size_to_fit += (pad_in_front ? Align(min_obj_size) : 0);
8792 #endif //SHORT_PLUGS
8794 int plug_power2 = index_of_set_bit (round_up_power2 (plug_size_to_fit + Align(min_obj_size)));
8796 uint8_t* new_address = 0;
8798 if (plug_power2 < base_power2)
8800 plug_power2 = base_power2;
8803 int chosen_power2 = plug_power2 - base_power2;
8805 for (i = chosen_power2; i < free_space_bucket_count; i++)
8807 if (free_space_buckets[i].count_fit != 0)
8814 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting plug len %Id (2^%d) using 2^%d free space",
8818 (chosen_power2 + base_power2)));
8820 assert (i < free_space_bucket_count);
8822 seg_free_space* bucket_free_space = free_space_buckets[chosen_power2].free_space;
8823 ptrdiff_t free_space_count = free_space_buckets[chosen_power2].count_fit;
8824 size_t new_free_space_size = 0;
8825 BOOL can_fit = FALSE;
8828 for (i = 0; i < free_space_count; i++)
8830 size_t free_space_size = 0;
8833 BOOL short_plugs_padding_p = FALSE;
8834 #endif //SHORT_PLUGS
8835 BOOL realign_padding_p = FALSE;
8837 if (bucket_free_space[i].is_plug)
8839 mark* m = (mark*)(bucket_free_space[i].start);
8840 uint8_t* plug_free_space_start = pinned_plug (m) - pinned_len (m);
8843 if ((pad_in_front & USE_PADDING_FRONT) &&
8844 (((plug_free_space_start - pin_allocation_context_start_region (m))==0) ||
8845 ((plug_free_space_start - pin_allocation_context_start_region (m))>=DESIRED_PLUG_LENGTH)))
8847 pad = Align (min_obj_size);
8848 short_plugs_padding_p = TRUE;
8850 #endif //SHORT_PLUGS
8852 if (!((old_loc == 0) || same_large_alignment_p (old_loc, plug_free_space_start+pad)))
8854 pad += switch_alignment_size (pad != 0);
8855 realign_padding_p = TRUE;
8858 plug_size = saved_plug_size + pad;
8860 free_space_size = pinned_len (m);
8861 new_address = pinned_plug (m) - pinned_len (m);
8863 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
8864 free_space_size == plug_size)
8866 new_free_space_size = free_space_size - plug_size;
8867 pinned_len (m) = new_free_space_size;
8868 #ifdef SIMPLE_DPRINTF
8869 dprintf (SEG_REUSE_LOG_0, ("[%d]FP: 0x%Ix->0x%Ix(%Ix)(%Ix), [0x%Ix (2^%d) -> [0x%Ix (2^%d)",
8876 index_of_set_bit (round_down_power2 (free_space_size)),
8877 (pinned_plug (m) - pinned_len (m)),
8878 index_of_set_bit (round_down_power2 (new_free_space_size))));
8879 #endif //SIMPLE_DPRINTF
8882 if (short_plugs_padding_p)
8884 pin_allocation_context_start_region (m) = plug_free_space_start;
8885 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
8887 #endif //SHORT_PLUGS
8889 if (realign_padding_p)
8891 set_node_realigned (old_loc);
8899 heap_segment* seg = (heap_segment*)(bucket_free_space[i].start);
8900 free_space_size = heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
8902 if (!((old_loc == 0) || same_large_alignment_p (old_loc, heap_segment_plan_allocated (seg))))
8904 pad = switch_alignment_size (FALSE);
8905 realign_padding_p = TRUE;
8908 plug_size = saved_plug_size + pad;
8910 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
8911 free_space_size == plug_size)
8913 new_address = heap_segment_plan_allocated (seg);
8914 new_free_space_size = free_space_size - plug_size;
8915 heap_segment_plan_allocated (seg) = new_address + plug_size;
8916 #ifdef SIMPLE_DPRINTF
8917 dprintf (SEG_REUSE_LOG_0, ("[%d]FS: 0x%Ix-> 0x%Ix(%Ix) (2^%d) -> 0x%Ix (2^%d)",
8922 index_of_set_bit (round_down_power2 (free_space_size)),
8923 heap_segment_plan_allocated (seg),
8924 index_of_set_bit (round_down_power2 (new_free_space_size))));
8925 #endif //SIMPLE_DPRINTF
8927 if (realign_padding_p)
8928 set_node_realigned (old_loc);
8942 assert (chosen_power2 == 0);
8952 assert ((chosen_power2 && (i == 0)) ||
8953 (!chosen_power2) && (i < free_space_count));
8956 int new_bucket_power2 = index_of_set_bit (round_down_power2 (new_free_space_size));
8958 if (new_bucket_power2 < base_power2)
8960 new_bucket_power2 = base_power2;
8963 move_bucket (chosen_power2, new_bucket_power2 - base_power2);
8972 if (free_space_buckets)
8974 delete [] free_space_buckets;
8976 if (seg_free_space_array)
8978 delete [] seg_free_space_array;
8984 #define marked(i) header(i)->IsMarked()
8985 #define set_marked(i) header(i)->SetMarked()
8986 #define clear_marked(i) header(i)->ClearMarked()
8987 #define pinned(i) header(i)->IsPinned()
8988 #define set_pinned(i) header(i)->SetPinned()
8989 #define clear_pinned(i) header(i)->GetHeader()->ClrGCBit();
8991 inline size_t my_get_size (Object* ob)
8993 MethodTable* mT = header(ob)->GetMethodTable();
8994 return (mT->GetBaseSize() +
8995 (mT->HasComponentSize() ?
8996 ((size_t)((CObjectHeader*)ob)->GetNumComponents() * mT->RawGetComponentSize()) : 0));
8999 //#define size(i) header(i)->GetSize()
9000 #define size(i) my_get_size (header(i))
9002 #define contain_pointers(i) header(i)->ContainsPointers()
9003 #ifdef COLLECTIBLE_CLASS
9004 #define contain_pointers_or_collectible(i) header(i)->ContainsPointersOrCollectible()
9006 #define get_class_object(i) GCToEEInterface::GetLoaderAllocatorObjectForGC((Object *)i)
9007 #define is_collectible(i) method_table(i)->Collectible()
9008 #else //COLLECTIBLE_CLASS
9009 #define contain_pointers_or_collectible(i) header(i)->ContainsPointers()
9010 #endif //COLLECTIBLE_CLASS
9012 #if defined (MARK_ARRAY) && defined (BACKGROUND_GC)
9014 void gc_heap::seg_clear_mark_array_bits_soh (heap_segment* seg)
9016 uint8_t* range_beg = 0;
9017 uint8_t* range_end = 0;
9018 if (bgc_mark_array_range (seg, FALSE, &range_beg, &range_end))
9020 clear_mark_array (range_beg, align_on_mark_word (range_end), FALSE
9021 #ifdef FEATURE_BASICFREEZE
9023 #endif // FEATURE_BASICFREEZE
9028 void gc_heap::clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9030 if ((start < background_saved_highest_address) &&
9031 (end > background_saved_lowest_address))
9033 start = max (start, background_saved_lowest_address);
9034 end = min (end, background_saved_highest_address);
9036 size_t start_mark_bit = mark_bit_of (start);
9037 size_t end_mark_bit = mark_bit_of (end);
9038 unsigned int startbit = mark_bit_bit (start_mark_bit);
9039 unsigned int endbit = mark_bit_bit (end_mark_bit);
9040 size_t startwrd = mark_bit_word (start_mark_bit);
9041 size_t endwrd = mark_bit_word (end_mark_bit);
9043 dprintf (3, ("Clearing all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
9044 (size_t)start, (size_t)start_mark_bit,
9045 (size_t)end, (size_t)end_mark_bit));
9047 unsigned int firstwrd = lowbits (~0, startbit);
9048 unsigned int lastwrd = highbits (~0, endbit);
9050 if (startwrd == endwrd)
9052 unsigned int wrd = firstwrd | lastwrd;
9053 mark_array[startwrd] &= wrd;
9057 // clear the first mark word.
9060 mark_array[startwrd] &= firstwrd;
9064 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
9066 mark_array[wrdtmp] = 0;
9069 // clear the last mark word.
9072 mark_array[endwrd] &= lastwrd;
9077 void gc_heap::bgc_clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9079 if ((start < background_saved_highest_address) &&
9080 (end > background_saved_lowest_address))
9082 start = max (start, background_saved_lowest_address);
9083 end = min (end, background_saved_highest_address);
9085 clear_batch_mark_array_bits (start, end);
9089 void gc_heap::clear_mark_array_by_objects (uint8_t* from, uint8_t* end, BOOL loh_p)
9091 dprintf (3, ("clearing mark array bits by objects for addr [%Ix,[%Ix",
9093 int align_const = get_alignment_constant (!loh_p);
9099 uint8_t* next_o = o + Align (size (o), align_const);
9101 if (background_object_marked (o, TRUE))
9103 dprintf (3, ("%Ix was marked by bgc, is now cleared", o));
9109 #endif //MARK_ARRAY && BACKGROUND_GC
9112 BOOL gc_heap::is_mark_set (uint8_t* o)
9117 #if defined (_MSC_VER) && defined (_TARGET_X86_)
9118 #pragma optimize("y", on) // Small critical routines, don't put in EBP frame
9119 #endif //_MSC_VER && _TARGET_X86_
9121 // return the generation number of an object.
9122 // It is assumed that the object is valid.
9123 //Note that this will return max_generation for a LOH object
9124 int gc_heap::object_gennum (uint8_t* o)
9126 if (in_range_for_segment (o, ephemeral_heap_segment) &&
9127 (o >= generation_allocation_start (generation_of (max_generation-1))))
9129 // in an ephemeral generation.
9130 for ( int i = 0; i < max_generation-1; i++)
9132 if ((o >= generation_allocation_start (generation_of (i))))
9135 return max_generation-1;
9139 return max_generation;
9143 int gc_heap::object_gennum_plan (uint8_t* o)
9145 if (in_range_for_segment (o, ephemeral_heap_segment))
9147 for (int i = 0; i <= max_generation-1; i++)
9149 uint8_t* plan_start = generation_plan_allocation_start (generation_of (i));
9150 if (plan_start && (o >= plan_start))
9156 return max_generation;
9159 #if defined(_MSC_VER) && defined(_TARGET_X86_)
9160 #pragma optimize("", on) // Go back to command line default optimizations
9161 #endif //_MSC_VER && _TARGET_X86_
9163 heap_segment* gc_heap::make_heap_segment (uint8_t* new_pages, size_t size, int h_number)
9165 size_t initial_commit = SEGMENT_INITIAL_COMMIT;
9167 //Commit the first page
9168 if (!virtual_alloc_commit_for_heap (new_pages, initial_commit, h_number))
9173 //overlay the heap_segment
9174 heap_segment* new_segment = (heap_segment*)new_pages;
9176 uint8_t* start = new_pages + segment_info_size;
9177 heap_segment_mem (new_segment) = start;
9178 heap_segment_used (new_segment) = start;
9179 heap_segment_reserved (new_segment) = new_pages + size;
9180 heap_segment_committed (new_segment) = new_pages + initial_commit;
9181 init_heap_segment (new_segment);
9182 dprintf (2, ("Creating heap segment %Ix", (size_t)new_segment));
9186 void gc_heap::init_heap_segment (heap_segment* seg)
9189 heap_segment_next (seg) = 0;
9190 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
9191 heap_segment_allocated (seg) = heap_segment_mem (seg);
9192 #ifdef BACKGROUND_GC
9193 heap_segment_background_allocated (seg) = 0;
9194 heap_segment_saved_bg_allocated (seg) = 0;
9195 #endif //BACKGROUND_GC
9198 //Releases the segment to the OS.
9199 // this is always called on one thread only so calling seg_table->remove is fine.
9200 void gc_heap::delete_heap_segment (heap_segment* seg, BOOL consider_hoarding)
9202 if (!heap_segment_loh_p (seg))
9204 //cleanup the brick table back to the empty value
9205 clear_brick_table (heap_segment_mem (seg), heap_segment_reserved (seg));
9208 if (consider_hoarding)
9210 assert ((heap_segment_mem (seg) - (uint8_t*)seg) <= ptrdiff_t(2*OS_PAGE_SIZE));
9211 size_t ss = (size_t) (heap_segment_reserved (seg) - (uint8_t*)seg);
9212 //Don't keep the big ones.
9213 if (ss <= INITIAL_ALLOC)
9215 dprintf (2, ("Hoarding segment %Ix", (size_t)seg));
9216 #ifdef BACKGROUND_GC
9217 // We don't need to clear the decommitted flag because when this segment is used
9218 // for a new segment the flags will be cleared.
9219 if (!heap_segment_decommitted_p (seg))
9220 #endif //BACKGROUND_GC
9222 decommit_heap_segment (seg);
9225 #ifdef SEG_MAPPING_TABLE
9226 seg_mapping_table_remove_segment (seg);
9227 #endif //SEG_MAPPING_TABLE
9229 heap_segment_next (seg) = segment_standby_list;
9230 segment_standby_list = seg;
9237 dprintf (2, ("h%d: del seg: [%Ix, %Ix[",
9238 heap_number, (size_t)seg,
9239 (size_t)(heap_segment_reserved (seg))));
9241 #ifdef BACKGROUND_GC
9242 ::record_changed_seg ((uint8_t*)seg, heap_segment_reserved (seg),
9243 settings.gc_index, current_bgc_state,
9245 decommit_mark_array_by_seg (seg);
9246 #endif //BACKGROUND_GC
9248 #ifdef SEG_MAPPING_TABLE
9249 seg_mapping_table_remove_segment (seg);
9250 #else //SEG_MAPPING_TABLE
9251 seg_table->remove ((uint8_t*)seg);
9252 #endif //SEG_MAPPING_TABLE
9254 release_segment (seg);
9258 //resets the pages beyond allocates size so they won't be swapped out and back in
9260 void gc_heap::reset_heap_segment_pages (heap_segment* seg)
9262 size_t page_start = align_on_page ((size_t)heap_segment_allocated (seg));
9263 size_t size = (size_t)heap_segment_committed (seg) - page_start;
9265 GCToOSInterface::VirtualReset((void*)page_start, size, false /* unlock */);
9268 void gc_heap::decommit_heap_segment_pages (heap_segment* seg,
9271 uint8_t* page_start = align_on_page (heap_segment_allocated(seg));
9272 size_t size = heap_segment_committed (seg) - page_start;
9273 extra_space = align_on_page (extra_space);
9274 if (size >= max ((extra_space + 2*OS_PAGE_SIZE), 100*OS_PAGE_SIZE))
9276 page_start += max(extra_space, 32*OS_PAGE_SIZE);
9277 size -= max (extra_space, 32*OS_PAGE_SIZE);
9279 GCToOSInterface::VirtualDecommit (page_start, size);
9280 dprintf (3, ("Decommitting heap segment [%Ix, %Ix[(%d)",
9282 (size_t)(page_start + size),
9284 heap_segment_committed (seg) = page_start;
9285 if (heap_segment_used (seg) > heap_segment_committed (seg))
9287 heap_segment_used (seg) = heap_segment_committed (seg);
9292 //decommit all pages except one or 2
9293 void gc_heap::decommit_heap_segment (heap_segment* seg)
9295 uint8_t* page_start = align_on_page (heap_segment_mem (seg));
9297 dprintf (3, ("Decommitting heap segment %Ix", (size_t)seg));
9299 #ifdef BACKGROUND_GC
9300 page_start += OS_PAGE_SIZE;
9301 #endif //BACKGROUND_GC
9303 size_t size = heap_segment_committed (seg) - page_start;
9304 GCToOSInterface::VirtualDecommit (page_start, size);
9306 //re-init the segment object
9307 heap_segment_committed (seg) = page_start;
9308 if (heap_segment_used (seg) > heap_segment_committed (seg))
9310 heap_segment_used (seg) = heap_segment_committed (seg);
9314 void gc_heap::clear_gen0_bricks()
9316 if (!gen0_bricks_cleared)
9318 gen0_bricks_cleared = TRUE;
9319 //initialize brick table for gen 0
9320 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
9321 b < brick_of (align_on_brick
9322 (heap_segment_allocated (ephemeral_heap_segment)));
9330 #ifdef BACKGROUND_GC
9331 void gc_heap::rearrange_small_heap_segments()
9333 heap_segment* seg = freeable_small_heap_segment;
9336 heap_segment* next_seg = heap_segment_next (seg);
9337 // TODO: we need to consider hoarding here.
9338 delete_heap_segment (seg, FALSE);
9341 freeable_small_heap_segment = 0;
9343 #endif //BACKGROUND_GC
9345 void gc_heap::rearrange_large_heap_segments()
9347 dprintf (2, ("deleting empty large segments"));
9348 heap_segment* seg = freeable_large_heap_segment;
9351 heap_segment* next_seg = heap_segment_next (seg);
9352 delete_heap_segment (seg, GCConfig::GetRetainVM());
9355 freeable_large_heap_segment = 0;
9358 void gc_heap::rearrange_heap_segments(BOOL compacting)
9361 generation_start_segment (generation_of (max_generation));
9363 heap_segment* prev_seg = 0;
9364 heap_segment* next_seg = 0;
9367 next_seg = heap_segment_next (seg);
9369 //link ephemeral segment when expanding
9370 if ((next_seg == 0) && (seg != ephemeral_heap_segment))
9372 seg->next = ephemeral_heap_segment;
9373 next_seg = heap_segment_next (seg);
9376 //re-used expanded heap segment
9377 if ((seg == ephemeral_heap_segment) && next_seg)
9379 heap_segment_next (prev_seg) = next_seg;
9380 heap_segment_next (seg) = 0;
9384 uint8_t* end_segment = (compacting ?
9385 heap_segment_plan_allocated (seg) :
9386 heap_segment_allocated (seg));
9387 // check if the segment was reached by allocation
9388 if ((end_segment == heap_segment_mem (seg))&&
9389 !heap_segment_read_only_p (seg))
9391 //if not, unthread and delete
9393 assert (seg != ephemeral_heap_segment);
9394 heap_segment_next (prev_seg) = next_seg;
9395 delete_heap_segment (seg, GCConfig::GetRetainVM());
9397 dprintf (2, ("Deleting heap segment %Ix", (size_t)seg));
9401 if (!heap_segment_read_only_p (seg))
9405 heap_segment_allocated (seg) =
9406 heap_segment_plan_allocated (seg);
9409 // reset the pages between allocated and committed.
9410 if (seg != ephemeral_heap_segment)
9412 decommit_heap_segment_pages (seg, 0);
9426 uint8_t* g_addresses [array_size+2]; // to get around the bug in GetWriteWatch
9428 #ifdef TIME_WRITE_WATCH
9429 static unsigned int tot_cycles = 0;
9430 #endif //TIME_WRITE_WATCH
9434 inline void gc_heap::verify_card_bundle_bits_set(size_t first_card_word, size_t last_card_word)
9437 for (size_t x = cardw_card_bundle (first_card_word); x < cardw_card_bundle (last_card_word); x++)
9439 if (!card_bundle_set_p (x))
9441 assert (!"Card bundle not set");
9442 dprintf (3, ("Card bundle %Ix not set", x));
9448 // Verifies that any bundles that are not set represent only cards that are not set.
9449 inline void gc_heap::verify_card_bundles()
9452 size_t lowest_card = card_word (card_of (lowest_address));
9453 size_t highest_card = card_word (card_of (highest_address));
9454 size_t cardb = cardw_card_bundle (lowest_card);
9455 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (highest_card));
9457 while (cardb < end_cardb)
9459 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb), lowest_card)];
9460 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1), highest_card)];
9462 if (card_bundle_set_p (cardb) == 0)
9464 // Verify that no card is set
9465 while (card_word < card_word_end)
9467 if (*card_word != 0)
9469 dprintf (3, ("gc: %d, Card word %Ix for address %Ix set, card_bundle %Ix clear",
9470 dd_collection_count (dynamic_data_of (0)),
9471 (size_t)(card_word-&card_table[0]),
9472 (size_t)(card_address ((size_t)(card_word-&card_table[0]) * card_word_width)), cardb));
9475 assert((*card_word)==0);
9485 // If card bundles are enabled, use write watch to find pages in the card table that have
9486 // been dirtied, and set the corresponding card bundle bits.
9487 void gc_heap::update_card_table_bundle()
9489 if (card_bundles_enabled())
9491 // The address of the card word containing the card representing the lowest heap address
9492 uint8_t* base_address = (uint8_t*)(&card_table[card_word (card_of (lowest_address))]);
9494 // The address of the card word containing the card representing the highest heap address
9495 uint8_t* high_address = (uint8_t*)(&card_table[card_word (card_of (highest_address))]);
9497 uint8_t* saved_base_address = base_address;
9498 uintptr_t bcount = array_size;
9499 size_t saved_region_size = align_on_page (high_address) - saved_base_address;
9503 size_t region_size = align_on_page (high_address) - base_address;
9505 dprintf (3,("Probing card table pages [%Ix, %Ix[", (size_t)base_address, (size_t)base_address+region_size));
9506 bool success = GCToOSInterface::GetWriteWatch(false /* resetState */,
9509 (void**)g_addresses,
9511 assert (success && "GetWriteWatch failed!");
9513 dprintf (3,("Found %d pages written", bcount));
9514 for (unsigned i = 0; i < bcount; i++)
9516 // Offset of the dirty page from the start of the card table (clamped to base_address)
9517 size_t bcardw = (uint32_t*)(max(g_addresses[i],base_address)) - &card_table[0];
9519 // Offset of the end of the page from the start of the card table (clamped to high addr)
9520 size_t ecardw = (uint32_t*)(min(g_addresses[i]+OS_PAGE_SIZE, high_address)) - &card_table[0];
9521 assert (bcardw >= card_word (card_of (g_gc_lowest_address)));
9523 // Set the card bundle bits representing the dirty card table page
9524 card_bundles_set (cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw)));
9525 dprintf (3,("Set Card bundle [%Ix, %Ix[", cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw))));
9527 verify_card_bundle_bits_set(bcardw, ecardw);
9530 if (bcount >= array_size)
9532 base_address = g_addresses [array_size-1] + OS_PAGE_SIZE;
9533 bcount = array_size;
9536 } while ((bcount >= array_size) && (base_address < high_address));
9538 // Now that we've updated the card bundle bits, reset the write-tracking state.
9539 GCToOSInterface::ResetWriteWatch (saved_base_address, saved_region_size);
9542 #endif //CARD_BUNDLE
9545 void gc_heap::reset_write_watch_for_gc_heap(void* base_address, size_t region_size)
9547 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9548 SoftwareWriteWatch::ClearDirty(base_address, region_size);
9549 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9550 GCToOSInterface::ResetWriteWatch(base_address, region_size);
9551 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9555 void gc_heap::get_write_watch_for_gc_heap(bool reset, void *base_address, size_t region_size, void** dirty_pages, uintptr_t* dirty_page_count_ref, bool is_runtime_suspended)
9557 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9558 SoftwareWriteWatch::GetDirty(base_address, region_size, dirty_pages, dirty_page_count_ref, reset, is_runtime_suspended);
9559 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9560 UNREFERENCED_PARAMETER(is_runtime_suspended);
9561 bool success = GCToOSInterface::GetWriteWatch(reset, base_address, region_size, dirty_pages, dirty_page_count_ref);
9563 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9566 const size_t ww_reset_quantum = 128*1024*1024;
9569 void gc_heap::switch_one_quantum()
9571 enable_preemptive ();
9572 GCToOSInterface::Sleep (1);
9573 disable_preemptive (true);
9576 void gc_heap::reset_ww_by_chunk (uint8_t* start_address, size_t total_reset_size)
9578 size_t reset_size = 0;
9579 size_t remaining_reset_size = 0;
9580 size_t next_reset_size = 0;
9582 while (reset_size != total_reset_size)
9584 remaining_reset_size = total_reset_size - reset_size;
9585 next_reset_size = ((remaining_reset_size >= ww_reset_quantum) ? ww_reset_quantum : remaining_reset_size);
9586 if (next_reset_size)
9588 reset_write_watch_for_gc_heap(start_address, next_reset_size);
9589 reset_size += next_reset_size;
9591 switch_one_quantum();
9595 assert (reset_size == total_reset_size);
9598 // This does a Sleep(1) for every reset ww_reset_quantum bytes of reset
9599 // we do concurrently.
9600 void gc_heap::switch_on_reset (BOOL concurrent_p, size_t* current_total_reset_size, size_t last_reset_size)
9604 *current_total_reset_size += last_reset_size;
9606 dprintf (2, ("reset %Id bytes so far", *current_total_reset_size));
9608 if (*current_total_reset_size > ww_reset_quantum)
9610 switch_one_quantum();
9612 *current_total_reset_size = 0;
9617 void gc_heap::reset_write_watch (BOOL concurrent_p)
9619 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9620 // Software write watch currently requires the runtime to be suspended during reset. See SoftwareWriteWatch::ClearDirty().
9621 assert(!concurrent_p);
9622 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9624 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
9626 PREFIX_ASSUME(seg != NULL);
9628 size_t reset_size = 0;
9629 size_t region_size = 0;
9631 dprintf (2, ("bgc lowest: %Ix, bgc highest: %Ix", background_saved_lowest_address, background_saved_highest_address));
9635 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9636 base_address = max (base_address, background_saved_lowest_address);
9638 uint8_t* high_address = 0;
9639 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
9640 high_address = min (high_address, background_saved_highest_address);
9642 if (base_address < high_address)
9644 region_size = high_address - base_address;
9646 #ifdef TIME_WRITE_WATCH
9647 unsigned int time_start = GetCycleCount32();
9648 #endif //TIME_WRITE_WATCH
9649 dprintf (3, ("h%d: soh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9650 //reset_ww_by_chunk (base_address, region_size);
9651 reset_write_watch_for_gc_heap(base_address, region_size);
9653 #ifdef TIME_WRITE_WATCH
9654 unsigned int time_stop = GetCycleCount32();
9655 tot_cycles += time_stop - time_start;
9656 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9657 time_stop - time_start, tot_cycles);
9658 #endif //TIME_WRITE_WATCH
9660 switch_on_reset (concurrent_p, &reset_size, region_size);
9663 seg = heap_segment_next_rw (seg);
9665 concurrent_print_time_delta ("CRWW soh");
9668 //concurrent_print_time_delta ("CRW soh");
9670 seg = heap_segment_rw (generation_start_segment (large_object_generation));
9672 PREFIX_ASSUME(seg != NULL);
9676 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9677 uint8_t* high_address = heap_segment_allocated (seg);
9679 base_address = max (base_address, background_saved_lowest_address);
9680 high_address = min (high_address, background_saved_highest_address);
9682 if (base_address < high_address)
9684 region_size = high_address - base_address;
9686 #ifdef TIME_WRITE_WATCH
9687 unsigned int time_start = GetCycleCount32();
9688 #endif //TIME_WRITE_WATCH
9689 dprintf (3, ("h%d: loh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9690 //reset_ww_by_chunk (base_address, region_size);
9691 reset_write_watch_for_gc_heap(base_address, region_size);
9693 #ifdef TIME_WRITE_WATCH
9694 unsigned int time_stop = GetCycleCount32();
9695 tot_cycles += time_stop - time_start;
9696 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9697 time_stop - time_start, tot_cycles);
9698 #endif //TIME_WRITE_WATCH
9700 switch_on_reset (concurrent_p, &reset_size, region_size);
9703 seg = heap_segment_next_rw (seg);
9705 concurrent_print_time_delta ("CRWW loh");
9708 #ifdef DEBUG_WRITE_WATCH
9709 debug_write_watch = (uint8_t**)~0;
9710 #endif //DEBUG_WRITE_WATCH
9713 #endif //WRITE_WATCH
9715 #ifdef BACKGROUND_GC
9716 void gc_heap::restart_vm()
9718 //assert (generation_allocation_pointer (youngest_generation) == 0);
9719 dprintf (3, ("Restarting EE"));
9720 STRESS_LOG0(LF_GC, LL_INFO10000, "Concurrent GC: Retarting EE\n");
9721 ee_proceed_event.Set();
9725 void fire_alloc_wait_event (alloc_wait_reason awr, BOOL begin_p)
9727 if (awr != awr_ignored)
9731 FIRE_EVENT(BGCAllocWaitBegin, awr);
9735 FIRE_EVENT(BGCAllocWaitEnd, awr);
9741 void gc_heap::fire_alloc_wait_event_begin (alloc_wait_reason awr)
9743 fire_alloc_wait_event (awr, TRUE);
9747 void gc_heap::fire_alloc_wait_event_end (alloc_wait_reason awr)
9749 fire_alloc_wait_event (awr, FALSE);
9751 #endif //BACKGROUND_GC
9752 void gc_heap::make_generation (generation& gen, heap_segment* seg, uint8_t* start, uint8_t* pointer)
9754 gen.allocation_start = start;
9755 gen.allocation_context.alloc_ptr = pointer;
9756 gen.allocation_context.alloc_limit = pointer;
9757 gen.allocation_context.alloc_bytes = 0;
9758 gen.allocation_context.alloc_bytes_loh = 0;
9759 gen.allocation_context_start_region = pointer;
9760 gen.start_segment = seg;
9761 gen.allocation_segment = seg;
9762 gen.plan_allocation_start = 0;
9763 gen.free_list_space = 0;
9764 gen.pinned_allocated = 0;
9765 gen.free_list_allocated = 0;
9766 gen.end_seg_allocated = 0;
9767 gen.condemned_allocated = 0;
9768 gen.free_obj_space = 0;
9769 gen.allocation_size = 0;
9770 gen.pinned_allocation_sweep_size = 0;
9771 gen.pinned_allocation_compact_size = 0;
9772 gen.allocate_end_seg_p = FALSE;
9773 gen.free_list_allocator.clear();
9775 #ifdef FREE_USAGE_STATS
9776 memset (gen.gen_free_spaces, 0, sizeof (gen.gen_free_spaces));
9777 memset (gen.gen_current_pinned_free_spaces, 0, sizeof (gen.gen_current_pinned_free_spaces));
9778 memset (gen.gen_plugs, 0, sizeof (gen.gen_plugs));
9779 #endif //FREE_USAGE_STATS
9782 void gc_heap::adjust_ephemeral_limits ()
9784 ephemeral_low = generation_allocation_start (generation_of (max_generation - 1));
9785 ephemeral_high = heap_segment_reserved (ephemeral_heap_segment);
9787 dprintf (3, ("new ephemeral low: %Ix new ephemeral high: %Ix",
9788 (size_t)ephemeral_low, (size_t)ephemeral_high))
9790 #ifndef MULTIPLE_HEAPS
9791 // This updates the write barrier helpers with the new info.
9792 stomp_write_barrier_ephemeral(ephemeral_low, ephemeral_high);
9793 #endif // MULTIPLE_HEAPS
9796 #if defined(TRACE_GC) || defined(GC_CONFIG_DRIVEN)
9797 FILE* CreateLogFile(const GCConfigStringHolder& temp_logfile_name, bool is_config)
9801 if (!temp_logfile_name.Get())
9806 char logfile_name[MAX_LONGPATH+1];
9807 uint32_t pid = GCToOSInterface::GetCurrentProcessId();
9808 const char* suffix = is_config ? ".config.log" : ".log";
9809 _snprintf_s(logfile_name, MAX_LONGPATH+1, _TRUNCATE, "%s.%d%s", temp_logfile_name.Get(), pid, suffix);
9810 logFile = fopen(logfile_name, "wb");
9813 #endif //TRACE_GC || GC_CONFIG_DRIVEN
9815 HRESULT gc_heap::initialize_gc (size_t segment_size,
9817 #ifdef MULTIPLE_HEAPS
9818 ,unsigned number_of_heaps
9819 #endif //MULTIPLE_HEAPS
9823 if (GCConfig::GetLogEnabled())
9825 gc_log = CreateLogFile(GCConfig::GetLogFile(), false);
9830 // GCLogFileSize in MBs.
9831 gc_log_file_size = static_cast<size_t>(GCConfig::GetLogFileSize());
9833 if (gc_log_file_size <= 0 || gc_log_file_size > 500)
9839 gc_log_lock.Initialize();
9840 gc_log_buffer = new (nothrow) uint8_t [gc_log_buffer_size];
9847 memset (gc_log_buffer, '*', gc_log_buffer_size);
9849 max_gc_buffers = gc_log_file_size * 1024 * 1024 / gc_log_buffer_size;
9853 #ifdef GC_CONFIG_DRIVEN
9854 if (GCConfig::GetConfigLogEnabled())
9856 gc_config_log = CreateLogFile(GCConfig::GetConfigLogFile(), true);
9858 if (gc_config_log == NULL)
9861 gc_config_log_buffer = new (nothrow) uint8_t [gc_config_log_buffer_size];
9862 if (!gc_config_log_buffer)
9864 fclose(gc_config_log);
9868 compact_ratio = static_cast<int>(GCConfig::GetCompactRatio());
9870 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
9871 cprintf (("%2s | %6s | %1s | %1s | %2s | %2s | %2s | %2s | %2s || %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s |",
9875 "C", // compaction (empty means sweeping), 'M' means it was mandatory, 'W' means it was not
9876 "EX", // heap expansion
9878 "BF", // best fit (if it indicates neither NF nor BF it means it had to acquire a new seg.
9881 "PreS", // short object before pinned plug
9882 "PostS", // short object after pinned plug
9883 "Merge", // merged pinned plugs
9884 "Conv", // converted to pinned plug
9885 "Pre", // plug before pinned plug but not after
9886 "Post", // plug after pinned plug but not before
9887 "PrPo", // plug both before and after pinned plug
9888 "PreP", // pre short object padded
9889 "PostP" // post short object padded
9892 #endif //GC_CONFIG_DRIVEN
9895 GCConfigStringHolder logFileName = GCConfig::GetMixLogFile();
9896 if (logFileName.Get() != nullptr)
9898 GCStatistics::logFileName = _strdup(logFileName.Get());
9899 GCStatistics::logFile = fopen(GCStatistics::logFileName, "a");
9900 if (!GCStatistics::logFile)
9907 HRESULT hres = S_OK;
9910 hardware_write_watch_api_supported();
9911 #ifdef BACKGROUND_GC
9912 if (can_use_write_watch_for_gc_heap() && GCConfig::GetConcurrentGC())
9914 gc_can_use_concurrent = true;
9915 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9916 virtual_alloc_hardware_write_watch = true;
9917 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9921 gc_can_use_concurrent = false;
9923 #endif //BACKGROUND_GC
9924 #endif //WRITE_WATCH
9926 #ifdef BACKGROUND_GC
9927 // leave the first page to contain only segment info
9928 // because otherwise we could need to revisit the first page frequently in
9930 segment_info_size = OS_PAGE_SIZE;
9932 segment_info_size = Align (sizeof (heap_segment), get_alignment_constant (FALSE));
9933 #endif //BACKGROUND_GC
9935 reserved_memory = 0;
9936 unsigned block_count;
9937 #ifdef MULTIPLE_HEAPS
9938 reserved_memory_limit = (segment_size + heap_size) * number_of_heaps;
9939 block_count = number_of_heaps;
9940 #else //MULTIPLE_HEAPS
9941 reserved_memory_limit = segment_size + heap_size;
9943 #endif //MULTIPLE_HEAPS
9945 if (!reserve_initial_memory(segment_size,heap_size,block_count))
9946 return E_OUTOFMEMORY;
9949 //check if we need to turn on card_bundles.
9950 #ifdef MULTIPLE_HEAPS
9951 // use INT64 arithmetic here because of possible overflow on 32p
9952 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*number_of_heaps;
9954 // use INT64 arithmetic here because of possible overflow on 32p
9955 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
9956 #endif //MULTIPLE_HEAPS
9958 if (can_use_write_watch_for_card_table() && reserved_memory >= th)
9960 settings.card_bundles = TRUE;
9964 settings.card_bundles = FALSE;
9966 #endif //CARD_BUNDLE
9968 settings.first_init();
9970 int latency_level_from_config = static_cast<int>(GCConfig::GetLatencyLevel());
9971 if (latency_level_from_config >= latency_level_first && latency_level_from_config <= latency_level_last)
9973 gc_heap::latency_level = static_cast<gc_latency_level>(latency_level_from_config);
9978 g_gc_card_table = make_card_table (g_gc_lowest_address, g_gc_highest_address);
9980 if (!g_gc_card_table)
9981 return E_OUTOFMEMORY;
9985 #ifdef MULTIPLE_HEAPS
9986 n_heaps = number_of_heaps;
9988 g_heaps = new (nothrow) gc_heap* [number_of_heaps];
9990 return E_OUTOFMEMORY;
9993 #pragma warning(push)
9994 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
9996 g_promoted = new (nothrow) size_t [number_of_heaps*16];
9997 g_bpromoted = new (nothrow) size_t [number_of_heaps*16];
9999 g_mark_stack_busy = new (nothrow) int[(number_of_heaps+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
10000 #endif //MH_SC_MARK
10002 #pragma warning(pop)
10003 #endif // _PREFAST_
10004 if (!g_promoted || !g_bpromoted)
10005 return E_OUTOFMEMORY;
10008 if (!g_mark_stack_busy)
10009 return E_OUTOFMEMORY;
10010 #endif //MH_SC_MARK
10012 if (!create_thread_support (number_of_heaps))
10013 return E_OUTOFMEMORY;
10015 if (!heap_select::init (number_of_heaps))
10016 return E_OUTOFMEMORY;
10018 #endif //MULTIPLE_HEAPS
10020 if (!init_semi_shared())
10028 //Initializes PER_HEAP_ISOLATED data members.
10030 gc_heap::init_semi_shared()
10034 // This is used for heap expansion - it's to fix exactly the start for gen 0
10035 // through (max_generation-1). When we expand the heap we allocate all these
10036 // gen starts at the beginning of the new ephemeral seg.
10037 eph_gen_starts_size = (Align (min_obj_size)) * max_generation;
10040 #ifdef MULTIPLE_HEAPS
10041 mark_list_size = min (150*1024, max (8192, soh_segment_size/(2*10*32)));
10042 g_mark_list = make_mark_list (mark_list_size*n_heaps);
10044 min_balance_threshold = alloc_quantum_balance_units * CLR_SIZE * 2;
10045 #ifdef PARALLEL_MARK_LIST_SORT
10046 g_mark_list_copy = make_mark_list (mark_list_size*n_heaps);
10047 if (!g_mark_list_copy)
10051 #endif //PARALLEL_MARK_LIST_SORT
10053 #else //MULTIPLE_HEAPS
10055 mark_list_size = max (8192, soh_segment_size/(64*32));
10056 g_mark_list = make_mark_list (mark_list_size);
10058 #endif //MULTIPLE_HEAPS
10060 dprintf (3, ("mark_list_size: %d", mark_list_size));
10068 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10069 if (!seg_mapping_table_init())
10071 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10073 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10074 seg_table = sorted_table::make_sorted_table();
10078 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10080 segment_standby_list = 0;
10082 if (!full_gc_approach_event.CreateManualEventNoThrow(FALSE))
10086 if (!full_gc_end_event.CreateManualEventNoThrow(FALSE))
10091 fgn_maxgen_percent = 0;
10092 fgn_loh_percent = 0;
10093 full_gc_approach_event_set = false;
10095 memset (full_gc_counts, 0, sizeof (full_gc_counts));
10098 should_expand_in_full_gc = FALSE;
10100 #ifdef FEATURE_LOH_COMPACTION
10101 loh_compaction_always_p = GCConfig::GetLOHCompactionMode() != 0;
10102 loh_compaction_mode = loh_compaction_default;
10103 #endif //FEATURE_LOH_COMPACTION
10105 #ifdef BACKGROUND_GC
10106 memset (ephemeral_fgc_counts, 0, sizeof (ephemeral_fgc_counts));
10107 bgc_alloc_spin_count = static_cast<uint32_t>(GCConfig::GetBGCSpinCount());
10108 bgc_alloc_spin = static_cast<uint32_t>(GCConfig::GetBGCSpin());
10111 int number_bgc_threads = 1;
10112 #ifdef MULTIPLE_HEAPS
10113 number_bgc_threads = n_heaps;
10114 #endif //MULTIPLE_HEAPS
10115 if (!create_bgc_threads_support (number_bgc_threads))
10120 #endif //BACKGROUND_GC
10122 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
10124 #ifdef GC_CONFIG_DRIVEN
10125 compact_or_sweep_gcs[0] = 0;
10126 compact_or_sweep_gcs[1] = 0;
10127 #endif //GC_CONFIG_DRIVEN
10130 short_plugs_pad_ratio = (double)DESIRED_PLUG_LENGTH / (double)(DESIRED_PLUG_LENGTH - Align (min_obj_size));
10131 #endif //SHORT_PLUGS
10139 if (full_gc_approach_event.IsValid())
10141 full_gc_approach_event.CloseEvent();
10143 if (full_gc_end_event.IsValid())
10145 full_gc_end_event.CloseEvent();
10152 gc_heap* gc_heap::make_gc_heap (
10153 #ifdef MULTIPLE_HEAPS
10156 #endif //MULTIPLE_HEAPS
10161 #ifdef MULTIPLE_HEAPS
10162 res = new (nothrow) gc_heap;
10166 res->vm_heap = vm_hp;
10167 res->alloc_context_count = 0;
10170 #ifdef PARALLEL_MARK_LIST_SORT
10171 res->mark_list_piece_start = new (nothrow) uint8_t**[n_heaps];
10172 if (!res->mark_list_piece_start)
10176 #pragma warning(push)
10177 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
10178 #endif // _PREFAST_
10179 res->mark_list_piece_end = new (nothrow) uint8_t**[n_heaps + 32]; // +32 is padding to reduce false sharing
10181 #pragma warning(pop)
10182 #endif // _PREFAST_
10184 if (!res->mark_list_piece_end)
10186 #endif //PARALLEL_MARK_LIST_SORT
10190 #endif //MULTIPLE_HEAPS
10192 if (res->init_gc_heap (
10193 #ifdef MULTIPLE_HEAPS
10195 #else //MULTIPLE_HEAPS
10197 #endif //MULTIPLE_HEAPS
10203 #ifdef MULTIPLE_HEAPS
10206 return (gc_heap*)1;
10207 #endif //MULTIPLE_HEAPS
10211 gc_heap::wait_for_gc_done(int32_t timeOut)
10213 bool cooperative_mode = enable_preemptive ();
10215 uint32_t dwWaitResult = NOERROR;
10217 gc_heap* wait_heap = NULL;
10218 while (gc_heap::gc_started)
10220 #ifdef MULTIPLE_HEAPS
10221 wait_heap = GCHeap::GetHeap(heap_select::select_heap(NULL, 0))->pGenGCHeap;
10222 dprintf(2, ("waiting for the gc_done_event on heap %d", wait_heap->heap_number));
10223 #endif // MULTIPLE_HEAPS
10226 PREFIX_ASSUME(wait_heap != NULL);
10227 #endif // _PREFAST_
10229 dwWaitResult = wait_heap->gc_done_event.Wait(timeOut, FALSE);
10231 disable_preemptive (cooperative_mode);
10233 return dwWaitResult;
10237 gc_heap::set_gc_done()
10239 enter_gc_done_event_lock();
10240 if (!gc_done_event_set)
10242 gc_done_event_set = true;
10243 dprintf (2, ("heap %d: setting gc_done_event", heap_number));
10244 gc_done_event.Set();
10246 exit_gc_done_event_lock();
10250 gc_heap::reset_gc_done()
10252 enter_gc_done_event_lock();
10253 if (gc_done_event_set)
10255 gc_done_event_set = false;
10256 dprintf (2, ("heap %d: resetting gc_done_event", heap_number));
10257 gc_done_event.Reset();
10259 exit_gc_done_event_lock();
10263 gc_heap::enter_gc_done_event_lock()
10265 uint32_t dwSwitchCount = 0;
10268 if (Interlocked::CompareExchange(&gc_done_event_lock, 0, -1) >= 0)
10270 while (gc_done_event_lock >= 0)
10272 if (g_num_processors > 1)
10274 int spin_count = 32 * g_num_processors;
10275 for (int j = 0; j < spin_count; j++)
10277 if (gc_done_event_lock < 0)
10279 YieldProcessor(); // indicate to the processor that we are spining
10281 if (gc_done_event_lock >= 0)
10282 GCToOSInterface::YieldThread(++dwSwitchCount);
10285 GCToOSInterface::YieldThread(++dwSwitchCount);
10292 gc_heap::exit_gc_done_event_lock()
10294 gc_done_event_lock = -1;
10297 #ifndef MULTIPLE_HEAPS
10299 #ifdef RECORD_LOH_STATE
10300 int gc_heap::loh_state_index = 0;
10301 gc_heap::loh_state_info gc_heap::last_loh_states[max_saved_loh_states];
10302 #endif //RECORD_LOH_STATE
10304 VOLATILE(int32_t) gc_heap::gc_done_event_lock;
10305 VOLATILE(bool) gc_heap::gc_done_event_set;
10306 GCEvent gc_heap::gc_done_event;
10307 #endif //!MULTIPLE_HEAPS
10308 VOLATILE(bool) gc_heap::internal_gc_done;
10310 void gc_heap::add_saved_spinlock_info (
10311 msl_enter_state enter_state,
10312 msl_take_state take_state)
10315 #ifdef SPINLOCK_HISTORY
10316 spinlock_info* current = &last_spinlock_info[spinlock_info_index];
10318 current->enter_state = enter_state;
10319 current->take_state = take_state;
10320 current->thread_id.SetToCurrentThread();
10322 spinlock_info_index++;
10324 assert (spinlock_info_index <= max_saved_spinlock_info);
10326 if (spinlock_info_index >= max_saved_spinlock_info)
10328 spinlock_info_index = 0;
10331 MAYBE_UNUSED_VAR(enter_state);
10332 MAYBE_UNUSED_VAR(take_state);
10333 #endif //SPINLOCK_HISTORY
10337 gc_heap::init_gc_heap (int h_number)
10339 #ifdef MULTIPLE_HEAPS
10343 #ifdef SPINLOCK_HISTORY
10344 spinlock_info_index = 0;
10345 memset (last_spinlock_info, 0, sizeof(last_spinlock_info));
10346 #endif //SPINLOCK_HISTORY
10348 // initialize per heap members.
10349 ephemeral_low = (uint8_t*)1;
10351 ephemeral_high = MAX_PTR;
10353 ephemeral_heap_segment = 0;
10355 freeable_large_heap_segment = 0;
10357 condemned_generation_num = 0;
10359 blocking_collection = FALSE;
10361 generation_skip_ratio = 100;
10363 mark_stack_tos = 0;
10365 mark_stack_bos = 0;
10367 mark_stack_array_length = 0;
10369 mark_stack_array = 0;
10371 verify_pinned_queue_p = FALSE;
10373 loh_pinned_queue_tos = 0;
10375 loh_pinned_queue_bos = 0;
10377 loh_pinned_queue_length = 0;
10379 loh_pinned_queue_decay = LOH_PIN_DECAY;
10381 loh_pinned_queue = 0;
10383 min_overflow_address = MAX_PTR;
10385 max_overflow_address = 0;
10387 gen0_bricks_cleared = FALSE;
10389 gen0_must_clear_bricks = 0;
10391 allocation_quantum = CLR_SIZE;
10393 more_space_lock = gc_lock;
10395 ro_segments_in_range = FALSE;
10397 loh_alloc_since_cg = 0;
10399 new_heap_segment = NULL;
10401 #ifdef RECORD_LOH_STATE
10402 loh_state_index = 0;
10403 #endif //RECORD_LOH_STATE
10404 #endif //MULTIPLE_HEAPS
10406 #ifdef MULTIPLE_HEAPS
10407 if (h_number > n_heaps)
10409 assert (!"Number of heaps exceeded");
10413 heap_number = h_number;
10414 #endif //MULTIPLE_HEAPS
10416 memset (&oom_info, 0, sizeof (oom_info));
10417 memset (&fgm_result, 0, sizeof (fgm_result));
10418 if (!gc_done_event.CreateManualEventNoThrow(FALSE))
10422 gc_done_event_lock = -1;
10423 gc_done_event_set = false;
10425 #ifndef SEG_MAPPING_TABLE
10426 if (!gc_heap::seg_table->ensure_space_for_insert ())
10430 #endif //!SEG_MAPPING_TABLE
10432 heap_segment* seg = get_initial_segment (soh_segment_size, h_number);
10436 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg),
10437 (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)),
10438 gc_etw_segment_small_object_heap);
10440 #ifdef SEG_MAPPING_TABLE
10441 seg_mapping_table_add_segment (seg, __this);
10442 #else //SEG_MAPPING_TABLE
10443 seg_table->insert ((uint8_t*)seg, sdelta);
10444 #endif //SEG_MAPPING_TABLE
10446 #ifdef MULTIPLE_HEAPS
10447 heap_segment_heap (seg) = this;
10448 #endif //MULTIPLE_HEAPS
10450 /* todo: Need a global lock for this */
10451 uint32_t* ct = &g_gc_card_table [card_word (card_of (g_gc_lowest_address))];
10452 own_card_table (ct);
10453 card_table = translate_card_table (ct);
10454 /* End of global lock */
10456 brick_table = card_table_brick_table (ct);
10457 highest_address = card_table_highest_address (ct);
10458 lowest_address = card_table_lowest_address (ct);
10461 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
10462 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
10463 card_table_card_bundle_table (ct));
10464 #endif //CARD_BUNDLE
10467 if (gc_can_use_concurrent)
10468 mark_array = translate_mark_array (card_table_mark_array (&g_gc_card_table[card_word (card_of (g_gc_lowest_address))]));
10471 #endif //MARK_ARRAY
10473 uint8_t* start = heap_segment_mem (seg);
10475 for (int i = 0; i < 1 + max_generation; i++)
10477 make_generation (generation_table [ (max_generation - i) ],
10479 generation_table [(max_generation - i)].gen_num = max_generation - i;
10480 start += Align (min_obj_size);
10483 heap_segment_allocated (seg) = start;
10484 alloc_allocated = start;
10485 heap_segment_used (seg) = start - plug_skew;
10487 ephemeral_heap_segment = seg;
10489 #ifndef SEG_MAPPING_TABLE
10490 if (!gc_heap::seg_table->ensure_space_for_insert ())
10494 #endif //!SEG_MAPPING_TABLE
10495 //Create the large segment generation
10496 heap_segment* lseg = get_initial_segment(min_loh_segment_size, h_number);
10499 lseg->flags |= heap_segment_flags_loh;
10501 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(lseg),
10502 (size_t)(heap_segment_reserved (lseg) - heap_segment_mem(lseg)),
10503 gc_etw_segment_large_object_heap);
10505 #ifdef SEG_MAPPING_TABLE
10506 seg_mapping_table_add_segment (lseg, __this);
10507 #else //SEG_MAPPING_TABLE
10508 seg_table->insert ((uint8_t*)lseg, sdelta);
10509 #endif //SEG_MAPPING_TABLE
10511 generation_table [max_generation].free_list_allocator = allocator(NUM_GEN2_ALIST, BASE_GEN2_ALIST, gen2_alloc_list);
10512 //assign the alloc_list for the large generation
10513 generation_table [max_generation+1].free_list_allocator = allocator(NUM_LOH_ALIST, BASE_LOH_ALIST, loh_alloc_list);
10514 generation_table [max_generation+1].gen_num = max_generation+1;
10515 make_generation (generation_table [max_generation+1],lseg, heap_segment_mem (lseg), 0);
10516 heap_segment_allocated (lseg) = heap_segment_mem (lseg) + Align (min_obj_size, get_alignment_constant (FALSE));
10517 heap_segment_used (lseg) = heap_segment_allocated (lseg) - plug_skew;
10519 for (int gen_num = 0; gen_num <= 1 + max_generation; gen_num++)
10521 generation* gen = generation_of (gen_num);
10522 make_unused_array (generation_allocation_start (gen), Align (min_obj_size));
10525 #ifdef MULTIPLE_HEAPS
10526 heap_segment_heap (lseg) = this;
10528 //initialize the alloc context heap
10529 generation_alloc_context (generation_of (0))->set_alloc_heap(vm_heap);
10531 //initialize the alloc context heap
10532 generation_alloc_context (generation_of (max_generation+1))->set_alloc_heap(vm_heap);
10534 #endif //MULTIPLE_HEAPS
10536 //Do this only once
10537 #ifdef MULTIPLE_HEAPS
10539 #endif //MULTIPLE_HEAPS
10541 #ifndef INTERIOR_POINTERS
10542 //set the brick_table for large objects
10543 //but default value is clearded
10544 //clear_brick_table ((uint8_t*)heap_segment_mem (lseg),
10545 // (uint8_t*)heap_segment_reserved (lseg));
10547 #else //INTERIOR_POINTERS
10549 //Because of the interior pointer business, we have to clear
10550 //the whole brick table
10551 //but the default value is cleared
10552 // clear_brick_table (lowest_address, highest_address);
10553 #endif //INTERIOR_POINTERS
10556 if (!init_dynamic_data())
10561 etw_allocation_running_amount[0] = 0;
10562 etw_allocation_running_amount[1] = 0;
10564 //needs to be done after the dynamic data has been initialized
10565 #ifndef MULTIPLE_HEAPS
10566 allocation_running_amount = dd_min_size (dynamic_data_of (0));
10567 #endif //!MULTIPLE_HEAPS
10569 fgn_last_alloc = dd_min_size (dynamic_data_of (0));
10571 mark* arr = new (nothrow) (mark [MARK_STACK_INITIAL_LENGTH]);
10575 make_mark_stack(arr);
10577 #ifdef BACKGROUND_GC
10578 freeable_small_heap_segment = 0;
10579 gchist_index_per_heap = 0;
10580 uint8_t** b_arr = new (nothrow) (uint8_t* [MARK_STACK_INITIAL_LENGTH]);
10584 make_background_mark_stack (b_arr);
10585 #endif //BACKGROUND_GC
10587 ephemeral_low = generation_allocation_start(generation_of(max_generation - 1));
10588 ephemeral_high = heap_segment_reserved(ephemeral_heap_segment);
10589 if (heap_number == 0)
10591 stomp_write_barrier_initialize(
10592 #ifdef MULTIPLE_HEAPS
10593 reinterpret_cast<uint8_t*>(1), reinterpret_cast<uint8_t*>(~0)
10595 ephemeral_low, ephemeral_high
10596 #endif //!MULTIPLE_HEAPS
10601 // why would we clear the mark array for this page? it should be cleared..
10602 // clear the first committed page
10603 //if(gc_can_use_concurrent)
10605 // clear_mark_array (align_lower_page (heap_segment_mem (seg)), heap_segment_committed (seg));
10607 #endif //MARK_ARRAY
10609 #ifdef MULTIPLE_HEAPS
10610 //register the heap in the heaps array
10612 if (!create_gc_thread ())
10615 g_heaps [heap_number] = this;
10617 #endif //MULTIPLE_HEAPS
10619 #ifdef FEATURE_PREMORTEM_FINALIZATION
10620 HRESULT hr = AllocateCFinalize(&finalize_queue);
10623 #endif // FEATURE_PREMORTEM_FINALIZATION
10625 max_free_space_items = MAX_NUM_FREE_SPACES;
10627 bestfit_seg = new (nothrow) seg_free_spaces (heap_number);
10634 if (!bestfit_seg->alloc())
10639 last_gc_before_oom = FALSE;
10641 #ifdef MULTIPLE_HEAPS
10643 #ifdef HEAP_ANALYZE
10645 heap_analyze_success = TRUE;
10647 internal_root_array = 0;
10649 internal_root_array_index = 0;
10651 internal_root_array_length = initial_internal_roots;
10655 current_obj_size = 0;
10657 #endif //HEAP_ANALYZE
10659 #endif // MULTIPLE_HEAPS
10661 #ifdef BACKGROUND_GC
10662 bgc_thread_id.Clear();
10664 if (!create_bgc_thread_support())
10669 bgc_alloc_lock = new (nothrow) exclusive_sync;
10670 if (!bgc_alloc_lock)
10675 bgc_alloc_lock->init();
10679 if (!recursive_gc_sync::init())
10683 bgc_thread_running = 0;
10685 bgc_threads_timeout_cs.Initialize();
10686 expanded_in_fgc = 0;
10687 current_bgc_state = bgc_not_in_process;
10688 background_soh_alloc_count = 0;
10689 background_loh_alloc_count = 0;
10690 bgc_overflow_count = 0;
10691 end_loh_size = dd_min_size (dynamic_data_of (max_generation + 1));
10692 #endif //BACKGROUND_GC
10694 #ifdef GC_CONFIG_DRIVEN
10695 memset (interesting_data_per_heap, 0, sizeof (interesting_data_per_heap));
10696 memset(compact_reasons_per_heap, 0, sizeof (compact_reasons_per_heap));
10697 memset(expand_mechanisms_per_heap, 0, sizeof (expand_mechanisms_per_heap));
10698 memset(interesting_mechanism_bits_per_heap, 0, sizeof (interesting_mechanism_bits_per_heap));
10699 #endif //GC_CONFIG_DRIVEN
10705 gc_heap::destroy_semi_shared()
10707 //TODO: will need to move this to per heap
10708 //#ifdef BACKGROUND_GC
10709 // if (c_mark_list)
10710 // delete c_mark_list;
10711 //#endif //BACKGROUND_GC
10715 delete g_mark_list;
10718 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10719 if (seg_mapping_table)
10720 delete seg_mapping_table;
10721 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10723 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10724 //destroy the segment map
10725 seg_table->delete_sorted_table();
10726 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10730 gc_heap::self_destroy()
10732 #ifdef BACKGROUND_GC
10734 #endif //BACKGROUND_GC
10736 if (gc_done_event.IsValid())
10738 gc_done_event.CloseEvent();
10741 // destroy every segment.
10742 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
10744 PREFIX_ASSUME(seg != NULL);
10746 heap_segment* next_seg;
10749 next_seg = heap_segment_next_rw (seg);
10750 delete_heap_segment (seg);
10754 seg = heap_segment_rw (generation_start_segment (generation_of (max_generation+1)));
10756 PREFIX_ASSUME(seg != NULL);
10760 next_seg = heap_segment_next_rw (seg);
10761 delete_heap_segment (seg);
10765 // get rid of the card table
10766 release_card_table (card_table);
10768 // destroy the mark stack
10769 delete mark_stack_array;
10771 #ifdef FEATURE_PREMORTEM_FINALIZATION
10772 if (finalize_queue)
10773 delete finalize_queue;
10774 #endif // FEATURE_PREMORTEM_FINALIZATION
10778 gc_heap::destroy_gc_heap(gc_heap* heap)
10780 heap->self_destroy();
10784 // Destroys resources owned by gc. It is assumed that a last GC has been performed and that
10785 // the finalizer queue has been drained.
10786 void gc_heap::shutdown_gc()
10788 destroy_semi_shared();
10790 #ifdef MULTIPLE_HEAPS
10791 //delete the heaps array
10793 destroy_thread_support();
10795 #endif //MULTIPLE_HEAPS
10796 //destroy seg_manager
10798 destroy_initial_memory();
10800 GCToOSInterface::Shutdown();
10804 BOOL gc_heap::size_fit_p (size_t size REQD_ALIGN_AND_OFFSET_DCL, uint8_t* alloc_pointer, uint8_t* alloc_limit,
10805 uint8_t* old_loc, int use_padding)
10807 BOOL already_padded = FALSE;
10809 if ((old_loc != 0) && (use_padding & USE_PADDING_FRONT))
10811 alloc_pointer = alloc_pointer + Align (min_obj_size);
10812 already_padded = TRUE;
10814 #endif //SHORT_PLUGS
10816 if (!((old_loc == 0) || same_large_alignment_p (old_loc, alloc_pointer)))
10817 size = size + switch_alignment_size (already_padded);
10819 #ifdef FEATURE_STRUCTALIGN
10820 alloc_pointer = StructAlign(alloc_pointer, requiredAlignment, alignmentOffset);
10821 #endif // FEATURE_STRUCTALIGN
10823 // in allocate_in_condemned_generation we can have this when we
10824 // set the alloc_limit to plan_allocated which could be less than
10826 if (alloc_limit < alloc_pointer)
10833 return (((size_t)(alloc_limit - alloc_pointer) >= (size + ((use_padding & USE_PADDING_TAIL)? Align(min_obj_size) : 0)))
10835 ||((!(use_padding & USE_PADDING_FRONT)) && ((alloc_pointer + size) == alloc_limit))
10836 #else //SHORT_PLUGS
10837 ||((alloc_pointer + size) == alloc_limit)
10838 #endif //SHORT_PLUGS
10843 assert (size == Align (min_obj_size));
10844 return ((size_t)(alloc_limit - alloc_pointer) >= size);
10849 BOOL gc_heap::a_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit,
10852 // We could have run into cases where this is true when alloc_allocated is the
10853 // the same as the seg committed.
10854 if (alloc_limit < alloc_pointer)
10859 return ((size_t)(alloc_limit - alloc_pointer) >= (size + Align(min_obj_size, align_const)));
10862 // Grow by committing more pages
10863 BOOL gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* high_address)
10865 assert (high_address <= heap_segment_reserved (seg));
10867 //return 0 if we are at the end of the segment.
10868 if (align_on_page (high_address) > heap_segment_reserved (seg))
10871 if (high_address <= heap_segment_committed (seg))
10874 size_t c_size = align_on_page ((size_t)(high_address - heap_segment_committed (seg)));
10875 c_size = max (c_size, 16*OS_PAGE_SIZE);
10876 c_size = min (c_size, (size_t)(heap_segment_reserved (seg) - heap_segment_committed (seg)));
10881 STRESS_LOG2(LF_GC, LL_INFO10000,
10882 "Growing heap_segment: %Ix high address: %Ix\n",
10883 (size_t)seg, (size_t)high_address);
10885 dprintf(3, ("Growing segment allocation %Ix %Ix", (size_t)heap_segment_committed(seg),c_size));
10887 if (!virtual_alloc_commit_for_heap(heap_segment_committed (seg), c_size, heap_number))
10889 dprintf(3, ("Cannot grow heap segment"));
10893 #ifndef BACKGROUND_GC
10894 clear_mark_array (heap_segment_committed (seg),
10895 heap_segment_committed (seg)+c_size, TRUE);
10896 #endif //BACKGROUND_GC
10897 #endif //MARK_ARRAY
10898 heap_segment_committed (seg) += c_size;
10899 STRESS_LOG1(LF_GC, LL_INFO10000, "New commit: %Ix",
10900 (size_t)heap_segment_committed (seg));
10902 assert (heap_segment_committed (seg) <= heap_segment_reserved (seg));
10904 assert (high_address <= heap_segment_committed (seg));
10910 int gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* allocated, uint8_t* old_loc, size_t size, BOOL pad_front_p REQD_ALIGN_AND_OFFSET_DCL)
10913 if ((old_loc != 0) && pad_front_p)
10915 allocated = allocated + Align (min_obj_size);
10917 #endif //SHORT_PLUGS
10919 if (!((old_loc == 0) || same_large_alignment_p (old_loc, allocated)))
10920 size = size + switch_alignment_size (FALSE);
10921 #ifdef FEATURE_STRUCTALIGN
10922 size_t pad = ComputeStructAlignPad(allocated, requiredAlignment, alignmentOffset);
10923 return grow_heap_segment (seg, allocated + pad + size);
10924 #else // FEATURE_STRUCTALIGN
10925 return grow_heap_segment (seg, allocated + size);
10926 #endif // FEATURE_STRUCTALIGN
10929 //used only in older generation allocation (i.e during gc).
10930 void gc_heap::adjust_limit (uint8_t* start, size_t limit_size, generation* gen,
10933 UNREFERENCED_PARAMETER(gennum);
10934 dprintf (3, ("gc Expanding segment allocation"));
10935 heap_segment* seg = generation_allocation_segment (gen);
10936 if ((generation_allocation_limit (gen) != start) || (start != heap_segment_plan_allocated (seg)))
10938 if (generation_allocation_limit (gen) == heap_segment_plan_allocated (seg))
10940 assert (generation_allocation_pointer (gen) >= heap_segment_mem (seg));
10941 assert (generation_allocation_pointer (gen) <= heap_segment_committed (seg));
10942 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
10946 uint8_t* hole = generation_allocation_pointer (gen);
10947 size_t size = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
10951 dprintf (3, ("filling up hole: %Ix, size %Ix", hole, size));
10952 size_t allocated_size = generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen);
10953 if (size >= Align (min_free_list))
10955 if (allocated_size < min_free_list)
10957 if (size >= (Align (min_free_list) + Align (min_obj_size)))
10959 //split hole into min obj + threadable free item
10960 make_unused_array (hole, min_obj_size);
10961 generation_free_obj_space (gen) += Align (min_obj_size);
10962 make_unused_array (hole + Align (min_obj_size), size - Align (min_obj_size));
10963 generation_free_list_space (gen) += size - Align (min_obj_size);
10964 generation_allocator(gen)->thread_item_front (hole + Align (min_obj_size),
10965 size - Align (min_obj_size));
10966 add_gen_free (gen->gen_num, (size - Align (min_obj_size)));
10970 dprintf (3, ("allocated size too small, can't put back rest on free list %Ix", allocated_size));
10971 make_unused_array (hole, size);
10972 generation_free_obj_space (gen) += size;
10977 dprintf (3, ("threading hole in front of free list"));
10978 make_unused_array (hole, size);
10979 generation_free_list_space (gen) += size;
10980 generation_allocator(gen)->thread_item_front (hole, size);
10981 add_gen_free (gen->gen_num, size);
10986 make_unused_array (hole, size);
10987 generation_free_obj_space (gen) += size;
10991 generation_allocation_pointer (gen) = start;
10992 generation_allocation_context_start_region (gen) = start;
10994 generation_allocation_limit (gen) = (start + limit_size);
10997 void verify_mem_cleared (uint8_t* start, size_t size)
10999 if (!Aligned (size))
11004 PTR_PTR curr_ptr = (PTR_PTR) start;
11005 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
11007 if (*(curr_ptr++) != 0)
11014 #if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
11015 void gc_heap::set_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11017 size_t start_mark_bit = mark_bit_of (start);
11018 size_t end_mark_bit = mark_bit_of (end);
11019 unsigned int startbit = mark_bit_bit (start_mark_bit);
11020 unsigned int endbit = mark_bit_bit (end_mark_bit);
11021 size_t startwrd = mark_bit_word (start_mark_bit);
11022 size_t endwrd = mark_bit_word (end_mark_bit);
11024 dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11025 (size_t)start, (size_t)start_mark_bit,
11026 (size_t)end, (size_t)end_mark_bit));
11028 unsigned int firstwrd = ~(lowbits (~0, startbit));
11029 unsigned int lastwrd = ~(highbits (~0, endbit));
11031 if (startwrd == endwrd)
11033 unsigned int wrd = firstwrd & lastwrd;
11034 mark_array[startwrd] |= wrd;
11038 // set the first mark word.
11041 mark_array[startwrd] |= firstwrd;
11045 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11047 mark_array[wrdtmp] = ~(unsigned int)0;
11050 // set the last mark word.
11053 mark_array[endwrd] |= lastwrd;
11057 // makes sure that the mark array bits between start and end are 0.
11058 void gc_heap::check_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11060 size_t start_mark_bit = mark_bit_of (start);
11061 size_t end_mark_bit = mark_bit_of (end);
11062 unsigned int startbit = mark_bit_bit (start_mark_bit);
11063 unsigned int endbit = mark_bit_bit (end_mark_bit);
11064 size_t startwrd = mark_bit_word (start_mark_bit);
11065 size_t endwrd = mark_bit_word (end_mark_bit);
11067 //dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11068 // (size_t)start, (size_t)start_mark_bit,
11069 // (size_t)end, (size_t)end_mark_bit));
11071 unsigned int firstwrd = ~(lowbits (~0, startbit));
11072 unsigned int lastwrd = ~(highbits (~0, endbit));
11074 if (startwrd == endwrd)
11076 unsigned int wrd = firstwrd & lastwrd;
11077 if (mark_array[startwrd] & wrd)
11079 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11081 mark_array [startwrd], mark_word_address (startwrd)));
11087 // set the first mark word.
11090 if (mark_array[startwrd] & firstwrd)
11092 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11093 firstwrd, startwrd,
11094 mark_array [startwrd], mark_word_address (startwrd)));
11101 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11103 if (mark_array[wrdtmp])
11105 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11107 mark_array [wrdtmp], mark_word_address (wrdtmp)));
11112 // set the last mark word.
11115 if (mark_array[endwrd] & lastwrd)
11117 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11119 mark_array [lastwrd], mark_word_address (lastwrd)));
11124 #endif //VERIFY_HEAP && BACKGROUND_GC
11126 allocator::allocator (unsigned int num_b, size_t fbs, alloc_list* b)
11128 assert (num_b < MAX_BUCKET_COUNT);
11129 num_buckets = num_b;
11130 frst_bucket_size = fbs;
11134 alloc_list& allocator::alloc_list_of (unsigned int bn)
11136 assert (bn < num_buckets);
11138 return first_bucket;
11140 return buckets [bn-1];
11143 size_t& allocator::alloc_list_damage_count_of (unsigned int bn)
11145 assert (bn < num_buckets);
11147 return first_bucket.alloc_list_damage_count();
11149 return buckets [bn-1].alloc_list_damage_count();
11152 void allocator::unlink_item (unsigned int bn, uint8_t* item, uint8_t* prev_item, BOOL use_undo_p)
11154 //unlink the free_item
11155 alloc_list* al = &alloc_list_of (bn);
11158 if (use_undo_p && (free_list_undo (prev_item) == UNDO_EMPTY))
11160 assert (item == free_list_slot (prev_item));
11161 free_list_undo (prev_item) = item;
11162 alloc_list_damage_count_of (bn)++;
11164 free_list_slot (prev_item) = free_list_slot(item);
11168 al->alloc_list_head() = (uint8_t*)free_list_slot(item);
11170 if (al->alloc_list_tail() == item)
11172 al->alloc_list_tail() = prev_item;
11176 void allocator::clear()
11178 for (unsigned int i = 0; i < num_buckets; i++)
11180 alloc_list_head_of (i) = 0;
11181 alloc_list_tail_of (i) = 0;
11185 //always thread to the end.
11186 void allocator::thread_free_item (uint8_t* item, uint8_t*& head, uint8_t*& tail)
11188 free_list_slot (item) = 0;
11189 free_list_undo (item) = UNDO_EMPTY;
11190 assert (item != head);
11196 //TODO: This shouldn't happen anymore - verify that's the case.
11197 //the following is necessary because the last free element
11198 //may have been truncated, and tail isn't updated.
11199 else if (free_list_slot (head) == 0)
11201 free_list_slot (head) = item;
11205 assert (item != tail);
11206 assert (free_list_slot(tail) == 0);
11207 free_list_slot (tail) = item;
11212 void allocator::thread_item (uint8_t* item, size_t size)
11214 size_t sz = frst_bucket_size;
11215 unsigned int a_l_number = 0;
11217 for (; a_l_number < (num_buckets-1); a_l_number++)
11225 alloc_list* al = &alloc_list_of (a_l_number);
11226 thread_free_item (item,
11227 al->alloc_list_head(),
11228 al->alloc_list_tail());
11231 void allocator::thread_item_front (uint8_t* item, size_t size)
11233 //find right free list
11234 size_t sz = frst_bucket_size;
11235 unsigned int a_l_number = 0;
11236 for (; a_l_number < (num_buckets-1); a_l_number++)
11244 alloc_list* al = &alloc_list_of (a_l_number);
11245 free_list_slot (item) = al->alloc_list_head();
11246 free_list_undo (item) = UNDO_EMPTY;
11248 if (al->alloc_list_tail() == 0)
11250 al->alloc_list_tail() = al->alloc_list_head();
11252 al->alloc_list_head() = item;
11253 if (al->alloc_list_tail() == 0)
11255 al->alloc_list_tail() = item;
11259 void allocator::copy_to_alloc_list (alloc_list* toalist)
11261 for (unsigned int i = 0; i < num_buckets; i++)
11263 toalist [i] = alloc_list_of (i);
11264 #ifdef FL_VERIFICATION
11265 uint8_t* free_item = alloc_list_head_of (i);
11270 free_item = free_list_slot (free_item);
11273 toalist[i].item_count = count;
11274 #endif //FL_VERIFICATION
11278 void allocator::copy_from_alloc_list (alloc_list* fromalist)
11280 BOOL repair_list = !discard_if_no_fit_p ();
11281 for (unsigned int i = 0; i < num_buckets; i++)
11283 size_t count = alloc_list_damage_count_of (i);
11284 alloc_list_of (i) = fromalist [i];
11285 assert (alloc_list_damage_count_of (i) == 0);
11289 //repair the the list
11290 //new items may have been added during the plan phase
11291 //items may have been unlinked.
11292 uint8_t* free_item = alloc_list_head_of (i);
11293 while (free_item && count)
11295 assert (((CObjectHeader*)free_item)->IsFree());
11296 if ((free_list_undo (free_item) != UNDO_EMPTY))
11299 free_list_slot (free_item) = free_list_undo (free_item);
11300 free_list_undo (free_item) = UNDO_EMPTY;
11303 free_item = free_list_slot (free_item);
11306 #ifdef FL_VERIFICATION
11307 free_item = alloc_list_head_of (i);
11308 size_t item_count = 0;
11312 free_item = free_list_slot (free_item);
11315 assert (item_count == alloc_list_of (i).item_count);
11316 #endif //FL_VERIFICATION
11319 uint8_t* tail_item = alloc_list_tail_of (i);
11320 assert ((tail_item == 0) || (free_list_slot (tail_item) == 0));
11325 void allocator::commit_alloc_list_changes()
11327 BOOL repair_list = !discard_if_no_fit_p ();
11330 for (unsigned int i = 0; i < num_buckets; i++)
11332 //remove the undo info from list.
11333 uint8_t* free_item = alloc_list_head_of (i);
11334 size_t count = alloc_list_damage_count_of (i);
11335 while (free_item && count)
11337 assert (((CObjectHeader*)free_item)->IsFree());
11339 if (free_list_undo (free_item) != UNDO_EMPTY)
11341 free_list_undo (free_item) = UNDO_EMPTY;
11345 free_item = free_list_slot (free_item);
11348 alloc_list_damage_count_of (i) = 0;
11353 void gc_heap::adjust_limit_clr (uint8_t* start, size_t limit_size,
11354 alloc_context* acontext, heap_segment* seg,
11355 int align_const, int gen_number)
11357 size_t aligned_min_obj_size = Align(min_obj_size, align_const);
11359 //probably should pass seg==0 for free lists.
11362 assert (heap_segment_used (seg) <= heap_segment_committed (seg));
11365 dprintf (3, ("Expanding segment allocation [%Ix, %Ix[", (size_t)start,
11366 (size_t)start + limit_size - aligned_min_obj_size));
11368 if ((acontext->alloc_limit != start) &&
11369 (acontext->alloc_limit + aligned_min_obj_size)!= start)
11371 uint8_t* hole = acontext->alloc_ptr;
11374 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
11375 dprintf (3, ("filling up hole [%Ix, %Ix[", (size_t)hole, (size_t)hole + size + Align (min_obj_size, align_const)));
11376 // when we are finishing an allocation from a free list
11377 // we know that the free area was Align(min_obj_size) larger
11378 acontext->alloc_bytes -= size;
11379 size_t free_obj_size = size + aligned_min_obj_size;
11380 make_unused_array (hole, free_obj_size);
11381 generation_free_obj_space (generation_of (gen_number)) += free_obj_size;
11383 acontext->alloc_ptr = start;
11387 // If the next alloc context is right up against the current one it means we are absorbing the min
11388 // object, so need to account for that.
11389 acontext->alloc_bytes += (start - acontext->alloc_limit);
11392 acontext->alloc_limit = (start + limit_size - aligned_min_obj_size);
11393 acontext->alloc_bytes += limit_size - ((gen_number < max_generation + 1) ? aligned_min_obj_size : 0);
11395 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
11398 AppDomain* alloc_appdomain = GetAppDomain();
11399 alloc_appdomain->RecordAllocBytes (limit_size, heap_number);
11401 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
11403 uint8_t* saved_used = 0;
11407 saved_used = heap_segment_used (seg);
11410 if (seg == ephemeral_heap_segment)
11412 //Sometimes the allocated size is advanced without clearing the
11413 //memory. Let's catch up here
11414 if (heap_segment_used (seg) < (alloc_allocated - plug_skew))
11417 #ifndef BACKGROUND_GC
11418 clear_mark_array (heap_segment_used (seg) + plug_skew, alloc_allocated);
11419 #endif //BACKGROUND_GC
11420 #endif //MARK_ARRAY
11421 heap_segment_used (seg) = alloc_allocated - plug_skew;
11424 #ifdef BACKGROUND_GC
11427 uint8_t* old_allocated = heap_segment_allocated (seg) - plug_skew - limit_size;
11428 #ifdef FEATURE_LOH_COMPACTION
11429 old_allocated -= Align (loh_padding_obj_size, align_const);
11430 #endif //FEATURE_LOH_COMPACTION
11432 assert (heap_segment_used (seg) >= old_allocated);
11434 #endif //BACKGROUND_GC
11436 (start - plug_skew + limit_size) <= heap_segment_used (seg))
11438 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear memory(1)", heap_number));
11439 add_saved_spinlock_info (me_release, mt_clr_mem);
11440 leave_spin_lock (&more_space_lock);
11441 dprintf (3, ("clearing memory at %Ix for %d bytes", (start - plug_skew), limit_size));
11442 memclr (start - plug_skew, limit_size);
11446 uint8_t* used = heap_segment_used (seg);
11447 heap_segment_used (seg) = start + limit_size - plug_skew;
11449 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear memory", heap_number));
11450 add_saved_spinlock_info (me_release, mt_clr_mem);
11451 leave_spin_lock (&more_space_lock);
11452 if ((start - plug_skew) < used)
11454 if (used != saved_used)
11459 dprintf (2, ("clearing memory before used at %Ix for %Id bytes",
11460 (start - plug_skew), (plug_skew + used - start)));
11461 memclr (start - plug_skew, used - (start - plug_skew));
11465 //this portion can be done after we release the lock
11466 if (seg == ephemeral_heap_segment)
11468 #ifdef FFIND_OBJECT
11469 if (gen0_must_clear_bricks > 0)
11471 //set the brick table to speed up find_object
11472 size_t b = brick_of (acontext->alloc_ptr);
11473 set_brick (b, acontext->alloc_ptr - brick_address (b));
11475 dprintf (3, ("Allocation Clearing bricks [%Ix, %Ix[",
11476 b, brick_of (align_on_brick (start + limit_size))));
11477 volatile short* x = &brick_table [b];
11478 short* end_x = &brick_table [brick_of (align_on_brick (start + limit_size))];
11480 for (;x < end_x;x++)
11484 #endif //FFIND_OBJECT
11486 gen0_bricks_cleared = FALSE;
11490 // verifying the memory is completely cleared.
11491 //verify_mem_cleared (start - plug_skew, limit_size);
11494 /* in order to make the allocator faster, allocate returns a
11495 * 0 filled object. Care must be taken to set the allocation limit to the
11496 * allocation pointer after gc
11499 size_t gc_heap::limit_from_size (size_t size, size_t room, int gen_number,
11502 size_t new_limit = new_allocation_limit ((size + Align (min_obj_size, align_const)),
11503 min (room,max (size + Align (min_obj_size, align_const),
11504 ((gen_number < max_generation+1) ?
11505 allocation_quantum :
11508 assert (new_limit >= (size + Align (min_obj_size, align_const)));
11509 dprintf (100, ("requested to allocate %Id bytes, actual size is %Id", size, new_limit));
11513 void gc_heap::handle_oom (int heap_num, oom_reason reason, size_t alloc_size,
11514 uint8_t* allocated, uint8_t* reserved)
11516 dprintf (1, ("total committed on the heap is %Id", get_total_committed_size()));
11518 UNREFERENCED_PARAMETER(heap_num);
11520 if (reason == oom_budget)
11522 alloc_size = dd_min_size (dynamic_data_of (0)) / 2;
11525 if ((reason == oom_budget) && ((!fgm_result.loh_p) && (fgm_result.fgm != fgm_no_failure)))
11527 // This means during the last GC we needed to reserve and/or commit more memory
11528 // but we couldn't. We proceeded with the GC and ended up not having enough
11529 // memory at the end. This is a legitimate OOM situtation. Otherwise we
11530 // probably made a mistake and didn't expand the heap when we should have.
11531 reason = oom_low_mem;
11534 oom_info.reason = reason;
11535 oom_info.allocated = allocated;
11536 oom_info.reserved = reserved;
11537 oom_info.alloc_size = alloc_size;
11538 oom_info.gc_index = settings.gc_index;
11539 oom_info.fgm = fgm_result.fgm;
11540 oom_info.size = fgm_result.size;
11541 oom_info.available_pagefile_mb = fgm_result.available_pagefile_mb;
11542 oom_info.loh_p = fgm_result.loh_p;
11544 fgm_result.fgm = fgm_no_failure;
11546 // Break early - before the more_space_lock is release so no other threads
11547 // could have allocated on the same heap when OOM happened.
11548 if (GCConfig::GetBreakOnOOM())
11550 GCToOSInterface::DebugBreak();
11554 #ifdef BACKGROUND_GC
11555 BOOL gc_heap::background_allowed_p()
11557 return ( gc_can_use_concurrent && ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)) );
11559 #endif //BACKGROUND_GC
11561 void gc_heap::check_for_full_gc (int gen_num, size_t size)
11563 BOOL should_notify = FALSE;
11564 // if we detect full gc because of the allocation budget specified this is TRUE;
11565 // it's FALSE if it's due to other factors.
11566 BOOL alloc_factor = TRUE;
11569 int n_initial = gen_num;
11570 BOOL local_blocking_collection = FALSE;
11571 BOOL local_elevation_requested = FALSE;
11572 int new_alloc_remain_percent = 0;
11574 if (full_gc_approach_event_set)
11579 if (gen_num != (max_generation + 1))
11581 gen_num = max_generation;
11584 dynamic_data* dd_full = dynamic_data_of (gen_num);
11585 ptrdiff_t new_alloc_remain = 0;
11586 uint32_t pct = ((gen_num == (max_generation + 1)) ? fgn_loh_percent : fgn_maxgen_percent);
11588 for (int gen_index = 0; gen_index <= (max_generation + 1); gen_index++)
11590 dprintf (2, ("FGN: h#%d: gen%d: %Id(%Id)",
11591 heap_number, gen_index,
11592 dd_new_allocation (dynamic_data_of (gen_index)),
11593 dd_desired_allocation (dynamic_data_of (gen_index))));
11596 // For small object allocations we only check every fgn_check_quantum bytes.
11597 if (n_initial == 0)
11599 dprintf (2, ("FGN: gen0 last recorded alloc: %Id", fgn_last_alloc));
11600 dynamic_data* dd_0 = dynamic_data_of (n_initial);
11601 if (((fgn_last_alloc - dd_new_allocation (dd_0)) < fgn_check_quantum) &&
11602 (dd_new_allocation (dd_0) >= 0))
11608 fgn_last_alloc = dd_new_allocation (dd_0);
11609 dprintf (2, ("FGN: gen0 last recorded alloc is now: %Id", fgn_last_alloc));
11612 // We don't consider the size that came from soh 'cause it doesn't contribute to the
11617 for (i = n+1; i <= max_generation; i++)
11619 if (get_new_allocation (i) <= 0)
11621 n = min (i, max_generation);
11627 dprintf (2, ("FGN: h#%d: gen%d budget exceeded", heap_number, n));
11628 if (gen_num == max_generation)
11630 // If it's small object heap we should first see if we will even be looking at gen2 budget
11631 // in the next GC or not. If not we should go directly to checking other factors.
11632 if (n < (max_generation - 1))
11634 goto check_other_factors;
11638 new_alloc_remain = dd_new_allocation (dd_full) - size;
11640 new_alloc_remain_percent = (int)(((float)(new_alloc_remain) / (float)dd_desired_allocation (dd_full)) * 100);
11642 dprintf (2, ("FGN: alloc threshold for gen%d is %d%%, current threshold is %d%%",
11643 gen_num, pct, new_alloc_remain_percent));
11645 if (new_alloc_remain_percent <= (int)pct)
11647 #ifdef BACKGROUND_GC
11648 // If background GC is enabled, we still want to check whether this will
11649 // be a blocking GC or not because we only want to notify when it's a
11650 // blocking full GC.
11651 if (background_allowed_p())
11653 goto check_other_factors;
11655 #endif //BACKGROUND_GC
11657 should_notify = TRUE;
11661 check_other_factors:
11663 dprintf (2, ("FGC: checking other factors"));
11664 n = generation_to_condemn (n,
11665 &local_blocking_collection,
11666 &local_elevation_requested,
11669 if (local_elevation_requested && (n == max_generation))
11671 if (settings.should_lock_elevation)
11673 int local_elevation_locked_count = settings.elevation_locked_count + 1;
11674 if (local_elevation_locked_count != 6)
11676 dprintf (2, ("FGN: lock count is %d - Condemning max_generation-1",
11677 local_elevation_locked_count));
11678 n = max_generation - 1;
11683 dprintf (2, ("FGN: we estimate gen%d will be collected", n));
11685 #ifdef BACKGROUND_GC
11686 // When background GC is enabled it decreases the accuracy of our predictability -
11687 // by the time the GC happens, we may not be under BGC anymore. If we try to
11688 // predict often enough it should be ok.
11689 if ((n == max_generation) &&
11690 (recursive_gc_sync::background_running_p()))
11692 n = max_generation - 1;
11693 dprintf (2, ("FGN: bgc - 1 instead of 2"));
11696 if ((n == max_generation) && !local_blocking_collection)
11698 if (!background_allowed_p())
11700 local_blocking_collection = TRUE;
11703 #endif //BACKGROUND_GC
11705 dprintf (2, ("FGN: we estimate gen%d will be collected: %s",
11707 (local_blocking_collection ? "blocking" : "background")));
11709 if ((n == max_generation) && local_blocking_collection)
11711 alloc_factor = FALSE;
11712 should_notify = TRUE;
11720 dprintf (2, ("FGN: gen%d detecting full GC approaching(%s) (GC#%d) (%Id%% left in gen%d)",
11722 (alloc_factor ? "alloc" : "other"),
11723 dd_collection_count (dynamic_data_of (0)),
11724 new_alloc_remain_percent,
11727 send_full_gc_notification (n_initial, alloc_factor);
11731 void gc_heap::send_full_gc_notification (int gen_num, BOOL due_to_alloc_p)
11733 if (!full_gc_approach_event_set)
11735 assert (full_gc_approach_event.IsValid());
11736 FIRE_EVENT(GCFullNotify_V1, gen_num, due_to_alloc_p);
11738 full_gc_end_event.Reset();
11739 full_gc_approach_event.Set();
11740 full_gc_approach_event_set = true;
11744 wait_full_gc_status gc_heap::full_gc_wait (GCEvent *event, int time_out_ms)
11746 if (fgn_maxgen_percent == 0)
11748 return wait_full_gc_na;
11751 uint32_t wait_result = user_thread_wait(event, FALSE, time_out_ms);
11753 if ((wait_result == WAIT_OBJECT_0) || (wait_result == WAIT_TIMEOUT))
11755 if (fgn_maxgen_percent == 0)
11757 return wait_full_gc_cancelled;
11760 if (wait_result == WAIT_OBJECT_0)
11762 #ifdef BACKGROUND_GC
11763 if (fgn_last_gc_was_concurrent)
11765 fgn_last_gc_was_concurrent = FALSE;
11766 return wait_full_gc_na;
11769 #endif //BACKGROUND_GC
11771 return wait_full_gc_success;
11776 return wait_full_gc_timeout;
11781 return wait_full_gc_failed;
11785 size_t gc_heap::get_full_compact_gc_count()
11787 return full_gc_counts[gc_type_compacting];
11790 // DTREVIEW - we should check this in dt_low_ephemeral_space_p
11793 BOOL gc_heap::short_on_end_of_seg (int gen_number,
11797 UNREFERENCED_PARAMETER(gen_number);
11798 uint8_t* allocated = heap_segment_allocated(seg);
11800 return (!a_size_fit_p (end_space_after_gc(),
11802 heap_segment_reserved (seg),
11807 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
11811 BOOL gc_heap::a_fit_free_list_p (int gen_number,
11813 alloc_context* acontext,
11816 BOOL can_fit = FALSE;
11817 generation* gen = generation_of (gen_number);
11818 allocator* gen_allocator = generation_allocator (gen);
11819 size_t sz_list = gen_allocator->first_bucket_size();
11820 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
11822 if ((size < sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
11824 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
11825 uint8_t* prev_free_item = 0;
11827 while (free_list != 0)
11829 dprintf (3, ("considering free list %Ix", (size_t)free_list));
11830 size_t free_list_size = unused_array_size (free_list);
11831 if ((size + Align (min_obj_size, align_const)) <= free_list_size)
11833 dprintf (3, ("Found adequate unused area: [%Ix, size: %Id",
11834 (size_t)free_list, free_list_size));
11836 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
11837 // We ask for more Align (min_obj_size)
11838 // to make sure that we can insert a free object
11839 // in adjust_limit will set the limit lower
11840 size_t limit = limit_from_size (size, free_list_size, gen_number, align_const);
11842 uint8_t* remain = (free_list + limit);
11843 size_t remain_size = (free_list_size - limit);
11844 if (remain_size >= Align(min_free_list, align_const))
11846 make_unused_array (remain, remain_size);
11847 gen_allocator->thread_item_front (remain, remain_size);
11848 assert (remain_size >= Align (min_obj_size, align_const));
11852 //absorb the entire free list
11853 limit += remain_size;
11855 generation_free_list_space (gen) -= limit;
11857 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
11862 else if (gen_allocator->discard_if_no_fit_p())
11864 assert (prev_free_item == 0);
11865 dprintf (3, ("couldn't use this free area, discarding"));
11866 generation_free_obj_space (gen) += free_list_size;
11868 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
11869 generation_free_list_space (gen) -= free_list_size;
11873 prev_free_item = free_list;
11875 free_list = free_list_slot (free_list);
11878 sz_list = sz_list * 2;
11885 #ifdef BACKGROUND_GC
11886 void gc_heap::bgc_loh_alloc_clr (uint8_t* alloc_start,
11888 alloc_context* acontext,
11894 make_unused_array (alloc_start, size);
11896 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
11899 AppDomain* alloc_appdomain = GetAppDomain();
11900 alloc_appdomain->RecordAllocBytes (size, heap_number);
11902 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
11904 size_t size_of_array_base = sizeof(ArrayBase);
11906 bgc_alloc_lock->loh_alloc_done_with_index (lock_index);
11908 // clear memory while not holding the lock.
11909 size_t size_to_skip = size_of_array_base;
11910 size_t size_to_clear = size - size_to_skip - plug_skew;
11911 size_t saved_size_to_clear = size_to_clear;
11914 uint8_t* end = alloc_start + size - plug_skew;
11915 uint8_t* used = heap_segment_used (seg);
11918 if ((alloc_start + size_to_skip) < used)
11920 size_to_clear = used - (alloc_start + size_to_skip);
11926 dprintf (2, ("bgc loh: setting used to %Ix", end));
11927 heap_segment_used (seg) = end;
11930 dprintf (2, ("bgc loh: used: %Ix, alloc: %Ix, end of alloc: %Ix, clear %Id bytes",
11931 used, alloc_start, end, size_to_clear));
11935 dprintf (2, ("bgc loh: [%Ix-[%Ix(%Id)", alloc_start, alloc_start+size, size));
11939 // since we filled in 0xcc for free object when we verify heap,
11940 // we need to make sure we clear those bytes.
11941 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
11943 if (size_to_clear < saved_size_to_clear)
11945 size_to_clear = saved_size_to_clear;
11948 #endif //VERIFY_HEAP
11950 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear large obj", heap_number));
11951 add_saved_spinlock_info (me_release, mt_clr_large_mem);
11952 leave_spin_lock (&more_space_lock);
11953 memclr (alloc_start + size_to_skip, size_to_clear);
11955 bgc_alloc_lock->loh_alloc_set (alloc_start);
11957 acontext->alloc_ptr = alloc_start;
11958 acontext->alloc_limit = (alloc_start + size - Align (min_obj_size, align_const));
11960 // need to clear the rest of the object before we hand it out.
11961 clear_unused_array(alloc_start, size);
11963 #endif //BACKGROUND_GC
11965 BOOL gc_heap::a_fit_free_list_large_p (size_t size,
11966 alloc_context* acontext,
11969 #ifdef BACKGROUND_GC
11970 wait_for_background_planning (awr_loh_alloc_during_plan);
11971 #endif //BACKGROUND_GC
11973 BOOL can_fit = FALSE;
11974 int gen_number = max_generation + 1;
11975 generation* gen = generation_of (gen_number);
11976 allocator* loh_allocator = generation_allocator (gen);
11978 #ifdef FEATURE_LOH_COMPACTION
11979 size_t loh_pad = Align (loh_padding_obj_size, align_const);
11980 #endif //FEATURE_LOH_COMPACTION
11982 #ifdef BACKGROUND_GC
11984 #endif //BACKGROUND_GC
11985 size_t sz_list = loh_allocator->first_bucket_size();
11986 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
11988 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
11990 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
11991 uint8_t* prev_free_item = 0;
11992 while (free_list != 0)
11994 dprintf (3, ("considering free list %Ix", (size_t)free_list));
11996 size_t free_list_size = unused_array_size(free_list);
11998 #ifdef FEATURE_LOH_COMPACTION
11999 if ((size + loh_pad) <= free_list_size)
12001 if (((size + Align (min_obj_size, align_const)) <= free_list_size)||
12002 (size == free_list_size))
12003 #endif //FEATURE_LOH_COMPACTION
12005 #ifdef BACKGROUND_GC
12006 cookie = bgc_alloc_lock->loh_alloc_set (free_list);
12007 #endif //BACKGROUND_GC
12009 //unlink the free_item
12010 loh_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12012 // Substract min obj size because limit_from_size adds it. Not needed for LOH
12013 size_t limit = limit_from_size (size - Align(min_obj_size, align_const), free_list_size,
12014 gen_number, align_const);
12016 #ifdef FEATURE_LOH_COMPACTION
12017 make_unused_array (free_list, loh_pad);
12019 free_list += loh_pad;
12020 free_list_size -= loh_pad;
12021 #endif //FEATURE_LOH_COMPACTION
12023 uint8_t* remain = (free_list + limit);
12024 size_t remain_size = (free_list_size - limit);
12025 if (remain_size != 0)
12027 assert (remain_size >= Align (min_obj_size, align_const));
12028 make_unused_array (remain, remain_size);
12030 if (remain_size >= Align(min_free_list, align_const))
12032 loh_thread_gap_front (remain, remain_size, gen);
12033 assert (remain_size >= Align (min_obj_size, align_const));
12037 generation_free_obj_space (gen) += remain_size;
12039 generation_free_list_space (gen) -= free_list_size;
12040 dprintf (3, ("found fit on loh at %Ix", free_list));
12041 #ifdef BACKGROUND_GC
12044 bgc_loh_alloc_clr (free_list, limit, acontext, align_const, cookie, FALSE, 0);
12047 #endif //BACKGROUND_GC
12049 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
12052 //fix the limit to compensate for adjust_limit_clr making it too short
12053 acontext->alloc_limit += Align (min_obj_size, align_const);
12057 prev_free_item = free_list;
12058 free_list = free_list_slot (free_list);
12061 sz_list = sz_list * 2;
12068 #pragma warning(default:4706)
12071 BOOL gc_heap::a_fit_segment_end_p (int gen_number,
12074 alloc_context* acontext,
12076 BOOL* commit_failed_p)
12078 *commit_failed_p = FALSE;
12080 #ifdef BACKGROUND_GC
12082 #endif //BACKGROUND_GC
12084 uint8_t*& allocated = ((gen_number == 0) ?
12086 heap_segment_allocated(seg));
12088 size_t pad = Align (min_obj_size, align_const);
12090 #ifdef FEATURE_LOH_COMPACTION
12091 if (gen_number == (max_generation + 1))
12093 pad += Align (loh_padding_obj_size, align_const);
12095 #endif //FEATURE_LOH_COMPACTION
12097 uint8_t* end = heap_segment_committed (seg) - pad;
12099 if (a_size_fit_p (size, allocated, end, align_const))
12101 limit = limit_from_size (size,
12103 gen_number, align_const);
12107 end = heap_segment_reserved (seg) - pad;
12109 if (a_size_fit_p (size, allocated, end, align_const))
12111 limit = limit_from_size (size,
12113 gen_number, align_const);
12114 if (grow_heap_segment (seg, allocated + limit))
12120 dprintf (2, ("can't grow segment, doing a full gc"));
12121 *commit_failed_p = TRUE;
12128 #ifdef BACKGROUND_GC
12129 if (gen_number != 0)
12131 cookie = bgc_alloc_lock->loh_alloc_set (allocated);
12133 #endif //BACKGROUND_GC
12135 uint8_t* old_alloc;
12136 old_alloc = allocated;
12137 #ifdef FEATURE_LOH_COMPACTION
12138 if (gen_number == (max_generation + 1))
12140 size_t loh_pad = Align (loh_padding_obj_size, align_const);
12141 make_unused_array (old_alloc, loh_pad);
12142 old_alloc += loh_pad;
12143 allocated += loh_pad;
12146 #endif //FEATURE_LOH_COMPACTION
12148 #if defined (VERIFY_HEAP) && defined (_DEBUG)
12149 ((void**) allocated)[-1] = 0; //clear the sync block
12150 #endif //VERIFY_HEAP && _DEBUG
12151 allocated += limit;
12153 dprintf (3, ("found fit at end of seg: %Ix", old_alloc));
12155 #ifdef BACKGROUND_GC
12158 bgc_loh_alloc_clr (old_alloc, limit, acontext, align_const, cookie, TRUE, seg);
12161 #endif //BACKGROUND_GC
12163 adjust_limit_clr (old_alloc, limit, acontext, seg, align_const, gen_number);
12173 BOOL gc_heap::loh_a_fit_segment_end_p (int gen_number,
12175 alloc_context* acontext,
12177 BOOL* commit_failed_p,
12180 *commit_failed_p = FALSE;
12181 heap_segment* seg = generation_allocation_segment (generation_of (gen_number));
12182 BOOL can_allocate_p = FALSE;
12186 if (a_fit_segment_end_p (gen_number, seg, (size - Align (min_obj_size, align_const)),
12187 acontext, align_const, commit_failed_p))
12189 acontext->alloc_limit += Align (min_obj_size, align_const);
12190 can_allocate_p = TRUE;
12195 if (*commit_failed_p)
12197 *oom_r = oom_cant_commit;
12202 seg = heap_segment_next_rw (seg);
12207 return can_allocate_p;
12210 #ifdef BACKGROUND_GC
12212 void gc_heap::wait_for_background (alloc_wait_reason awr)
12214 dprintf (2, ("BGC is already in progress, waiting for it to finish"));
12215 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to wait for bgc done", heap_number));
12216 add_saved_spinlock_info (me_release, mt_wait_bgc);
12217 leave_spin_lock (&more_space_lock);
12218 background_gc_wait (awr);
12219 enter_spin_lock (&more_space_lock);
12220 add_saved_spinlock_info (me_acquire, mt_wait_bgc);
12221 dprintf (SPINLOCK_LOG, ("[%d]Emsl after waiting for bgc done", heap_number));
12224 void gc_heap::wait_for_bgc_high_memory (alloc_wait_reason awr)
12226 if (recursive_gc_sync::background_running_p())
12228 uint32_t memory_load;
12229 get_memory_info (&memory_load);
12230 if (memory_load >= 95)
12232 dprintf (GTC_LOG, ("high mem - wait for BGC to finish, wait reason: %d", awr));
12233 wait_for_background (awr);
12238 #endif //BACKGROUND_GC
12240 // We request to trigger an ephemeral GC but we may get a full compacting GC.
12241 // return TRUE if that's the case.
12242 BOOL gc_heap::trigger_ephemeral_gc (gc_reason gr)
12244 #ifdef BACKGROUND_GC
12245 wait_for_bgc_high_memory (awr_loh_oos_bgc);
12246 #endif //BACKGROUND_GC
12248 BOOL did_full_compact_gc = FALSE;
12250 dprintf (2, ("triggering a gen1 GC"));
12251 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12252 vm_heap->GarbageCollectGeneration(max_generation - 1, gr);
12254 #ifdef MULTIPLE_HEAPS
12255 enter_spin_lock (&more_space_lock);
12256 add_saved_spinlock_info (me_acquire, mt_t_eph_gc);
12257 dprintf (SPINLOCK_LOG, ("[%d]Emsl after a GC", heap_number));
12258 #endif //MULTIPLE_HEAPS
12260 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12262 if (current_full_compact_gc_count > last_full_compact_gc_count)
12264 dprintf (2, ("attempted to trigger an ephemeral GC and got a full compacting GC"));
12265 did_full_compact_gc = TRUE;
12268 return did_full_compact_gc;
12271 BOOL gc_heap::soh_try_fit (int gen_number,
12273 alloc_context* acontext,
12275 BOOL* commit_failed_p,
12276 BOOL* short_seg_end_p)
12278 BOOL can_allocate = TRUE;
12279 if (short_seg_end_p)
12281 *short_seg_end_p = FALSE;
12284 can_allocate = a_fit_free_list_p (gen_number, size, acontext, align_const);
12287 if (short_seg_end_p)
12289 *short_seg_end_p = short_on_end_of_seg (gen_number, ephemeral_heap_segment, align_const);
12291 // If the caller doesn't care, we always try to fit at the end of seg;
12292 // otherwise we would only try if we are actually not short at end of seg.
12293 if (!short_seg_end_p || !(*short_seg_end_p))
12295 can_allocate = a_fit_segment_end_p (gen_number, ephemeral_heap_segment, size,
12296 acontext, align_const, commit_failed_p);
12300 return can_allocate;
12303 BOOL gc_heap::allocate_small (int gen_number,
12305 alloc_context* acontext,
12308 #if defined (BACKGROUND_GC) && !defined (MULTIPLE_HEAPS)
12309 if (recursive_gc_sync::background_running_p())
12311 background_soh_alloc_count++;
12312 if ((background_soh_alloc_count % bgc_alloc_spin_count) == 0)
12314 add_saved_spinlock_info (me_release, mt_alloc_small);
12315 dprintf (SPINLOCK_LOG, ("[%d]spin Lmsl", heap_number));
12316 leave_spin_lock (&more_space_lock);
12317 bool cooperative_mode = enable_preemptive ();
12318 GCToOSInterface::Sleep (bgc_alloc_spin);
12319 disable_preemptive (cooperative_mode);
12320 enter_spin_lock (&more_space_lock);
12321 add_saved_spinlock_info (me_acquire, mt_alloc_small);
12322 dprintf (SPINLOCK_LOG, ("[%d]spin Emsl", heap_number));
12326 //GCToOSInterface::YieldThread (0);
12329 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
12331 gc_reason gr = reason_oos_soh;
12332 oom_reason oom_r = oom_no_failure;
12334 // No variable values should be "carried over" from one state to the other.
12335 // That's why there are local variable for each state
12337 allocation_state soh_alloc_state = a_state_start;
12339 // If we can get a new seg it means allocation will succeed.
12342 dprintf (3, ("[h%d]soh state is %s", heap_number, allocation_state_str[soh_alloc_state]));
12343 switch (soh_alloc_state)
12345 case a_state_can_allocate:
12346 case a_state_cant_allocate:
12350 case a_state_start:
12352 soh_alloc_state = a_state_try_fit;
12355 case a_state_try_fit:
12357 BOOL commit_failed_p = FALSE;
12358 BOOL can_use_existing_p = FALSE;
12360 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12361 align_const, &commit_failed_p,
12363 soh_alloc_state = (can_use_existing_p ?
12364 a_state_can_allocate :
12366 a_state_trigger_full_compact_gc :
12367 a_state_trigger_ephemeral_gc));
12370 case a_state_try_fit_after_bgc:
12372 BOOL commit_failed_p = FALSE;
12373 BOOL can_use_existing_p = FALSE;
12374 BOOL short_seg_end_p = FALSE;
12376 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12377 align_const, &commit_failed_p,
12379 soh_alloc_state = (can_use_existing_p ?
12380 a_state_can_allocate :
12382 a_state_trigger_2nd_ephemeral_gc :
12383 a_state_trigger_full_compact_gc));
12386 case a_state_try_fit_after_cg:
12388 BOOL commit_failed_p = FALSE;
12389 BOOL can_use_existing_p = FALSE;
12390 BOOL short_seg_end_p = FALSE;
12392 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12393 align_const, &commit_failed_p,
12395 if (short_seg_end_p)
12397 soh_alloc_state = a_state_cant_allocate;
12398 oom_r = oom_budget;
12402 if (can_use_existing_p)
12404 soh_alloc_state = a_state_can_allocate;
12408 #ifdef MULTIPLE_HEAPS
12409 if (!commit_failed_p)
12411 // some other threads already grabbed the more space lock and allocated
12412 // so we should attempt an ephemeral GC again.
12413 assert (heap_segment_allocated (ephemeral_heap_segment) < alloc_allocated);
12414 soh_alloc_state = a_state_trigger_ephemeral_gc;
12417 #endif //MULTIPLE_HEAPS
12419 assert (commit_failed_p);
12420 soh_alloc_state = a_state_cant_allocate;
12421 oom_r = oom_cant_commit;
12427 case a_state_check_and_wait_for_bgc:
12429 BOOL bgc_in_progress_p = FALSE;
12430 BOOL did_full_compacting_gc = FALSE;
12432 bgc_in_progress_p = check_and_wait_for_bgc (awr_gen0_oos_bgc, &did_full_compacting_gc);
12433 soh_alloc_state = (did_full_compacting_gc ?
12434 a_state_try_fit_after_cg :
12435 a_state_try_fit_after_bgc);
12438 case a_state_trigger_ephemeral_gc:
12440 BOOL commit_failed_p = FALSE;
12441 BOOL can_use_existing_p = FALSE;
12442 BOOL short_seg_end_p = FALSE;
12443 BOOL bgc_in_progress_p = FALSE;
12444 BOOL did_full_compacting_gc = FALSE;
12446 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12447 if (did_full_compacting_gc)
12449 soh_alloc_state = a_state_try_fit_after_cg;
12453 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12454 align_const, &commit_failed_p,
12456 #ifdef BACKGROUND_GC
12457 bgc_in_progress_p = recursive_gc_sync::background_running_p();
12458 #endif //BACKGROUND_GC
12460 if (short_seg_end_p)
12462 soh_alloc_state = (bgc_in_progress_p ?
12463 a_state_check_and_wait_for_bgc :
12464 a_state_trigger_full_compact_gc);
12466 if (fgn_maxgen_percent)
12468 dprintf (2, ("FGN: doing last GC before we throw OOM"));
12469 send_full_gc_notification (max_generation, FALSE);
12474 if (can_use_existing_p)
12476 soh_alloc_state = a_state_can_allocate;
12480 #ifdef MULTIPLE_HEAPS
12481 if (!commit_failed_p)
12483 // some other threads already grabbed the more space lock and allocated
12484 // so we should attempt an ephemeral GC again.
12485 assert (heap_segment_allocated (ephemeral_heap_segment) < alloc_allocated);
12486 soh_alloc_state = a_state_trigger_ephemeral_gc;
12489 #endif //MULTIPLE_HEAPS
12491 soh_alloc_state = a_state_trigger_full_compact_gc;
12492 if (fgn_maxgen_percent)
12494 dprintf (2, ("FGN: failed to commit, doing full compacting GC"));
12495 send_full_gc_notification (max_generation, FALSE);
12503 case a_state_trigger_2nd_ephemeral_gc:
12505 BOOL commit_failed_p = FALSE;
12506 BOOL can_use_existing_p = FALSE;
12507 BOOL short_seg_end_p = FALSE;
12508 BOOL did_full_compacting_gc = FALSE;
12511 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12513 if (did_full_compacting_gc)
12515 soh_alloc_state = a_state_try_fit_after_cg;
12519 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12520 align_const, &commit_failed_p,
12522 if (short_seg_end_p || commit_failed_p)
12524 soh_alloc_state = a_state_trigger_full_compact_gc;
12528 assert (can_use_existing_p);
12529 soh_alloc_state = a_state_can_allocate;
12534 case a_state_trigger_full_compact_gc:
12536 BOOL got_full_compacting_gc = FALSE;
12538 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r);
12539 soh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
12544 assert (!"Invalid state!");
12551 if (soh_alloc_state == a_state_cant_allocate)
12553 assert (oom_r != oom_no_failure);
12554 handle_oom (heap_number,
12557 heap_segment_allocated (ephemeral_heap_segment),
12558 heap_segment_reserved (ephemeral_heap_segment));
12560 dprintf (SPINLOCK_LOG, ("[%d]Lmsl for oom", heap_number));
12561 add_saved_spinlock_info (me_release, mt_alloc_small_cant);
12562 leave_spin_lock (&more_space_lock);
12565 return (soh_alloc_state == a_state_can_allocate);
12568 #ifdef BACKGROUND_GC
12570 void gc_heap::wait_for_background_planning (alloc_wait_reason awr)
12572 while (current_c_gc_state == c_gc_state_planning)
12574 dprintf (3, ("lh state planning, cannot allocate"));
12576 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to wait for bgc plan", heap_number));
12577 add_saved_spinlock_info (me_release, mt_wait_bgc_plan);
12578 leave_spin_lock (&more_space_lock);
12579 background_gc_wait_lh (awr);
12580 enter_spin_lock (&more_space_lock);
12581 add_saved_spinlock_info (me_acquire, mt_wait_bgc_plan);
12582 dprintf (SPINLOCK_LOG, ("[%d]Emsl after waiting for bgc plan", heap_number));
12584 assert ((current_c_gc_state == c_gc_state_free) ||
12585 (current_c_gc_state == c_gc_state_marking));
12588 BOOL gc_heap::bgc_loh_should_allocate()
12590 size_t min_gc_size = dd_min_size(dynamic_data_of (max_generation + 1));
12592 if ((bgc_begin_loh_size + bgc_loh_size_increased) < (min_gc_size * 10))
12597 if (((bgc_begin_loh_size / end_loh_size) >= 2) || (bgc_loh_size_increased >= bgc_begin_loh_size))
12599 if ((bgc_begin_loh_size / end_loh_size) > 2)
12601 dprintf (3, ("alloc-ed too much before bgc started"));
12605 dprintf (3, ("alloc-ed too much after bgc started"));
12611 bgc_alloc_spin_loh = (uint32_t)(((float)bgc_loh_size_increased / (float)bgc_begin_loh_size) * 10);
12615 #endif //BACKGROUND_GC
12617 size_t gc_heap::get_large_seg_size (size_t size)
12619 size_t default_seg_size = min_loh_segment_size;
12620 #ifdef SEG_MAPPING_TABLE
12621 size_t align_size = default_seg_size;
12622 #else //SEG_MAPPING_TABLE
12623 size_t align_size = default_seg_size / 2;
12624 #endif //SEG_MAPPING_TABLE
12625 int align_const = get_alignment_constant (FALSE);
12626 size_t large_seg_size = align_on_page (
12627 max (default_seg_size,
12628 ((size + 2 * Align(min_obj_size, align_const) + OS_PAGE_SIZE +
12629 align_size) / align_size * align_size)));
12630 return large_seg_size;
12633 BOOL gc_heap::loh_get_new_seg (generation* gen,
12636 BOOL* did_full_compact_gc,
12639 UNREFERENCED_PARAMETER(gen);
12640 UNREFERENCED_PARAMETER(align_const);
12642 *did_full_compact_gc = FALSE;
12644 size_t seg_size = get_large_seg_size (size);
12646 heap_segment* new_seg = get_large_segment (seg_size, did_full_compact_gc);
12650 loh_alloc_since_cg += seg_size;
12657 return (new_seg != 0);
12660 BOOL gc_heap::retry_full_compact_gc (size_t size)
12662 size_t seg_size = get_large_seg_size (size);
12664 if (loh_alloc_since_cg >= (2 * (uint64_t)seg_size))
12669 #ifdef MULTIPLE_HEAPS
12670 uint64_t total_alloc_size = 0;
12671 for (int i = 0; i < n_heaps; i++)
12673 total_alloc_size += g_heaps[i]->loh_alloc_since_cg;
12676 if (total_alloc_size >= (2 * (uint64_t)seg_size))
12680 #endif //MULTIPLE_HEAPS
12685 BOOL gc_heap::check_and_wait_for_bgc (alloc_wait_reason awr,
12686 BOOL* did_full_compact_gc)
12688 BOOL bgc_in_progress = FALSE;
12689 *did_full_compact_gc = FALSE;
12690 #ifdef BACKGROUND_GC
12691 if (recursive_gc_sync::background_running_p())
12693 bgc_in_progress = TRUE;
12694 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12695 wait_for_background (awr);
12696 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12697 if (current_full_compact_gc_count > last_full_compact_gc_count)
12699 *did_full_compact_gc = TRUE;
12702 #endif //BACKGROUND_GC
12704 return bgc_in_progress;
12707 BOOL gc_heap::loh_try_fit (int gen_number,
12709 alloc_context* acontext,
12711 BOOL* commit_failed_p,
12714 BOOL can_allocate = TRUE;
12716 if (!a_fit_free_list_large_p (size, acontext, align_const))
12718 can_allocate = loh_a_fit_segment_end_p (gen_number, size,
12719 acontext, align_const,
12720 commit_failed_p, oom_r);
12722 #ifdef BACKGROUND_GC
12723 if (can_allocate && recursive_gc_sync::background_running_p())
12725 bgc_loh_size_increased += size;
12727 #endif //BACKGROUND_GC
12729 #ifdef BACKGROUND_GC
12732 if (recursive_gc_sync::background_running_p())
12734 bgc_loh_allocated_in_free += size;
12737 #endif //BACKGROUND_GC
12739 return can_allocate;
12742 BOOL gc_heap::trigger_full_compact_gc (gc_reason gr,
12745 BOOL did_full_compact_gc = FALSE;
12747 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12749 // Set this so the next GC will be a full compacting GC.
12750 if (!last_gc_before_oom)
12752 last_gc_before_oom = TRUE;
12755 #ifdef BACKGROUND_GC
12756 if (recursive_gc_sync::background_running_p())
12758 wait_for_background ((gr == reason_oos_soh) ? awr_gen0_oos_bgc : awr_loh_oos_bgc);
12759 dprintf (2, ("waited for BGC - done"));
12761 #endif //BACKGROUND_GC
12763 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12764 if (current_full_compact_gc_count > last_full_compact_gc_count)
12766 dprintf (3, ("a full compacting GC triggered while waiting for BGC (%d->%d)", last_full_compact_gc_count, current_full_compact_gc_count));
12767 assert (current_full_compact_gc_count > last_full_compact_gc_count);
12768 did_full_compact_gc = TRUE;
12772 dprintf (3, ("h%d full GC", heap_number));
12773 vm_heap->GarbageCollectGeneration(max_generation, gr);
12775 #ifdef MULTIPLE_HEAPS
12776 enter_spin_lock (&more_space_lock);
12777 dprintf (SPINLOCK_LOG, ("[%d]Emsl after full gc", heap_number));
12778 add_saved_spinlock_info (me_acquire, mt_t_full_gc);
12779 #endif //MULTIPLE_HEAPS
12781 current_full_compact_gc_count = get_full_compact_gc_count();
12783 if (current_full_compact_gc_count == last_full_compact_gc_count)
12785 dprintf (2, ("attempted to trigger a full compacting GC but didn't get it"));
12786 // We requested a full GC but didn't get because of the elevation logic
12787 // which means we should fail.
12788 *oom_r = oom_unproductive_full_gc;
12792 dprintf (3, ("h%d: T full compacting GC (%d->%d)",
12794 last_full_compact_gc_count,
12795 current_full_compact_gc_count));
12797 assert (current_full_compact_gc_count > last_full_compact_gc_count);
12798 did_full_compact_gc = TRUE;
12802 return did_full_compact_gc;
12805 #ifdef RECORD_LOH_STATE
12806 void gc_heap::add_saved_loh_state (allocation_state loh_state_to_save, EEThreadId thread_id)
12808 // When the state is can_allocate we already have released the more
12809 // space lock. So we are not logging states here since this code
12810 // is not thread safe.
12811 if (loh_state_to_save != a_state_can_allocate)
12813 last_loh_states[loh_state_index].alloc_state = loh_state_to_save;
12814 last_loh_states[loh_state_index].thread_id = thread_id;
12817 if (loh_state_index == max_saved_loh_states)
12819 loh_state_index = 0;
12822 assert (loh_state_index < max_saved_loh_states);
12825 #endif //RECORD_LOH_STATE
12827 BOOL gc_heap::allocate_large (int gen_number,
12829 alloc_context* acontext,
12832 #ifdef BACKGROUND_GC
12833 if (recursive_gc_sync::background_running_p() && (current_c_gc_state != c_gc_state_planning))
12835 background_loh_alloc_count++;
12836 //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0)
12838 if (bgc_loh_should_allocate())
12840 if (!bgc_alloc_spin_loh)
12842 add_saved_spinlock_info (me_release, mt_alloc_large);
12843 dprintf (SPINLOCK_LOG, ("[%d]spin Lmsl loh", heap_number));
12844 leave_spin_lock (&more_space_lock);
12845 bool cooperative_mode = enable_preemptive ();
12846 GCToOSInterface::YieldThread (bgc_alloc_spin_loh);
12847 disable_preemptive (cooperative_mode);
12848 enter_spin_lock (&more_space_lock);
12849 add_saved_spinlock_info (me_acquire, mt_alloc_large);
12850 dprintf (SPINLOCK_LOG, ("[%d]spin Emsl loh", heap_number));
12855 wait_for_background (awr_loh_alloc_during_bgc);
12859 #endif //BACKGROUND_GC
12861 gc_reason gr = reason_oos_loh;
12862 generation* gen = generation_of (gen_number);
12863 oom_reason oom_r = oom_no_failure;
12864 size_t current_full_compact_gc_count = 0;
12866 // No variable values should be "carried over" from one state to the other.
12867 // That's why there are local variable for each state
12868 allocation_state loh_alloc_state = a_state_start;
12869 #ifdef RECORD_LOH_STATE
12870 EEThreadId current_thread_id;
12871 current_thread_id.SetToCurrentThread();
12872 #endif //RECORD_LOH_STATE
12874 // If we can get a new seg it means allocation will succeed.
12877 dprintf (3, ("[h%d]loh state is %s", heap_number, allocation_state_str[loh_alloc_state]));
12879 #ifdef RECORD_LOH_STATE
12880 add_saved_loh_state (loh_alloc_state, current_thread_id);
12881 #endif //RECORD_LOH_STATE
12882 switch (loh_alloc_state)
12884 case a_state_can_allocate:
12885 case a_state_cant_allocate:
12889 case a_state_start:
12891 loh_alloc_state = a_state_try_fit;
12894 case a_state_try_fit:
12896 BOOL commit_failed_p = FALSE;
12897 BOOL can_use_existing_p = FALSE;
12899 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12900 align_const, &commit_failed_p, &oom_r);
12901 loh_alloc_state = (can_use_existing_p ?
12902 a_state_can_allocate :
12904 a_state_trigger_full_compact_gc :
12905 a_state_acquire_seg));
12906 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12909 case a_state_try_fit_new_seg:
12911 BOOL commit_failed_p = FALSE;
12912 BOOL can_use_existing_p = FALSE;
12914 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12915 align_const, &commit_failed_p, &oom_r);
12916 // Even after we got a new seg it doesn't necessarily mean we can allocate,
12917 // another LOH allocating thread could have beat us to acquire the msl so
12918 // we need to try again.
12919 loh_alloc_state = (can_use_existing_p ? a_state_can_allocate : a_state_try_fit);
12920 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12923 case a_state_try_fit_new_seg_after_cg:
12925 BOOL commit_failed_p = FALSE;
12926 BOOL can_use_existing_p = FALSE;
12928 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12929 align_const, &commit_failed_p, &oom_r);
12930 // Even after we got a new seg it doesn't necessarily mean we can allocate,
12931 // another LOH allocating thread could have beat us to acquire the msl so
12932 // we need to try again. However, if we failed to commit, which means we
12933 // did have space on the seg, we bail right away 'cause we already did a
12934 // full compacting GC.
12935 loh_alloc_state = (can_use_existing_p ?
12936 a_state_can_allocate :
12938 a_state_cant_allocate :
12939 a_state_acquire_seg_after_cg));
12940 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12943 case a_state_try_fit_no_seg:
12945 BOOL commit_failed_p = FALSE;
12946 BOOL can_use_existing_p = FALSE;
12948 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12949 align_const, &commit_failed_p, &oom_r);
12950 loh_alloc_state = (can_use_existing_p ? a_state_can_allocate : a_state_cant_allocate);
12951 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12952 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
12955 case a_state_try_fit_after_cg:
12957 BOOL commit_failed_p = FALSE;
12958 BOOL can_use_existing_p = FALSE;
12960 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12961 align_const, &commit_failed_p, &oom_r);
12962 loh_alloc_state = (can_use_existing_p ?
12963 a_state_can_allocate :
12965 a_state_cant_allocate :
12966 a_state_acquire_seg_after_cg));
12967 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12970 case a_state_try_fit_after_bgc:
12972 BOOL commit_failed_p = FALSE;
12973 BOOL can_use_existing_p = FALSE;
12975 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12976 align_const, &commit_failed_p, &oom_r);
12977 loh_alloc_state = (can_use_existing_p ?
12978 a_state_can_allocate :
12980 a_state_trigger_full_compact_gc :
12981 a_state_acquire_seg_after_bgc));
12982 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12985 case a_state_acquire_seg:
12987 BOOL can_get_new_seg_p = FALSE;
12988 BOOL did_full_compacting_gc = FALSE;
12990 current_full_compact_gc_count = get_full_compact_gc_count();
12992 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
12993 loh_alloc_state = (can_get_new_seg_p ?
12994 a_state_try_fit_new_seg :
12995 (did_full_compacting_gc ?
12996 a_state_check_retry_seg :
12997 a_state_check_and_wait_for_bgc));
13000 case a_state_acquire_seg_after_cg:
13002 BOOL can_get_new_seg_p = FALSE;
13003 BOOL did_full_compacting_gc = FALSE;
13005 current_full_compact_gc_count = get_full_compact_gc_count();
13007 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13008 // Since we release the msl before we try to allocate a seg, other
13009 // threads could have allocated a bunch of segments before us so
13010 // we might need to retry.
13011 loh_alloc_state = (can_get_new_seg_p ?
13012 a_state_try_fit_new_seg_after_cg :
13013 a_state_check_retry_seg);
13016 case a_state_acquire_seg_after_bgc:
13018 BOOL can_get_new_seg_p = FALSE;
13019 BOOL did_full_compacting_gc = FALSE;
13021 current_full_compact_gc_count = get_full_compact_gc_count();
13023 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13024 loh_alloc_state = (can_get_new_seg_p ?
13025 a_state_try_fit_new_seg :
13026 (did_full_compacting_gc ?
13027 a_state_check_retry_seg :
13028 a_state_trigger_full_compact_gc));
13029 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13032 case a_state_check_and_wait_for_bgc:
13034 BOOL bgc_in_progress_p = FALSE;
13035 BOOL did_full_compacting_gc = FALSE;
13037 if (fgn_maxgen_percent)
13039 dprintf (2, ("FGN: failed to acquire seg, may need to do a full blocking GC"));
13040 send_full_gc_notification (max_generation, FALSE);
13043 bgc_in_progress_p = check_and_wait_for_bgc (awr_loh_oos_bgc, &did_full_compacting_gc);
13044 loh_alloc_state = (!bgc_in_progress_p ?
13045 a_state_trigger_full_compact_gc :
13046 (did_full_compacting_gc ?
13047 a_state_try_fit_after_cg :
13048 a_state_try_fit_after_bgc));
13051 case a_state_trigger_full_compact_gc:
13053 BOOL got_full_compacting_gc = FALSE;
13055 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r);
13056 loh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
13057 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13060 case a_state_check_retry_seg:
13062 BOOL should_retry_gc = retry_full_compact_gc (size);
13063 BOOL should_retry_get_seg = FALSE;
13064 if (!should_retry_gc)
13066 size_t last_full_compact_gc_count = current_full_compact_gc_count;
13067 current_full_compact_gc_count = get_full_compact_gc_count();
13069 if (current_full_compact_gc_count > (last_full_compact_gc_count + 1))
13071 should_retry_get_seg = TRUE;
13075 loh_alloc_state = (should_retry_gc ?
13076 a_state_trigger_full_compact_gc :
13077 (should_retry_get_seg ?
13078 a_state_acquire_seg_after_cg :
13079 a_state_cant_allocate));
13080 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13085 assert (!"Invalid state!");
13092 if (loh_alloc_state == a_state_cant_allocate)
13094 assert (oom_r != oom_no_failure);
13095 handle_oom (heap_number,
13101 add_saved_spinlock_info (me_release, mt_alloc_large_cant);
13102 dprintf (SPINLOCK_LOG, ("[%d]Lmsl for loh oom", heap_number));
13103 leave_spin_lock (&more_space_lock);
13106 return (loh_alloc_state == a_state_can_allocate);
13109 int gc_heap::try_allocate_more_space (alloc_context* acontext, size_t size,
13112 if (gc_heap::gc_started)
13114 wait_for_gc_done();
13118 #ifdef SYNCHRONIZATION_STATS
13119 int64_t msl_acquire_start = GCToOSInterface::QueryPerformanceCounter();
13120 #endif //SYNCHRONIZATION_STATS
13121 enter_spin_lock (&more_space_lock);
13122 add_saved_spinlock_info (me_acquire, mt_try_alloc);
13123 dprintf (SPINLOCK_LOG, ("[%d]Emsl for alloc", heap_number));
13124 #ifdef SYNCHRONIZATION_STATS
13125 int64_t msl_acquire = GCToOSInterface::QueryPerformanceCounter() - msl_acquire_start;
13126 total_msl_acquire += msl_acquire;
13127 num_msl_acquired++;
13128 if (msl_acquire > 200)
13130 num_high_msl_acquire++;
13134 num_low_msl_acquire++;
13136 #endif //SYNCHRONIZATION_STATS
13139 // We are commenting this out 'cause we don't see the point - we already
13140 // have checked gc_started when we were acquiring the msl - no need to check
13141 // again. This complicates the logic in bgc_suspend_EE 'cause that one would
13142 // need to release msl which causes all sorts of trouble.
13143 if (gc_heap::gc_started)
13145 #ifdef SYNCHRONIZATION_STATS
13147 #endif //SYNCHRONIZATION_STATS
13148 BOOL fStress = (g_pConfig->GetGCStressLevel() & GCConfig::GCSTRESS_TRANSITION) != 0;
13151 //Rendez vous early (MP scaling issue)
13152 //dprintf (1, ("[%d]waiting for gc", heap_number));
13153 wait_for_gc_done();
13154 #ifdef MULTIPLE_HEAPS
13156 #endif //MULTIPLE_HEAPS
13161 dprintf (3, ("requested to allocate %d bytes on gen%d", size, gen_number));
13163 int align_const = get_alignment_constant (gen_number != (max_generation+1));
13165 if (fgn_maxgen_percent)
13167 check_for_full_gc (gen_number, size);
13170 if (!(new_allocation_allowed (gen_number)))
13172 if (fgn_maxgen_percent && (gen_number == 0))
13174 // We only check gen0 every so often, so take this opportunity to check again.
13175 check_for_full_gc (gen_number, size);
13178 #ifdef BACKGROUND_GC
13179 wait_for_bgc_high_memory (awr_gen0_alloc);
13180 #endif //BACKGROUND_GC
13182 #ifdef SYNCHRONIZATION_STATS
13184 #endif //SYNCHRONIZATION_STATS
13185 dprintf (/*100*/ 2, ("running out of budget on gen%d, gc", gen_number));
13187 if (!settings.concurrent || (gen_number == 0))
13189 vm_heap->GarbageCollectGeneration (0, ((gen_number == 0) ? reason_alloc_soh : reason_alloc_loh));
13190 #ifdef MULTIPLE_HEAPS
13191 enter_spin_lock (&more_space_lock);
13192 add_saved_spinlock_info (me_acquire, mt_try_budget);
13193 dprintf (SPINLOCK_LOG, ("[%d]Emsl out budget", heap_number));
13194 #endif //MULTIPLE_HEAPS
13198 BOOL can_allocate = ((gen_number == 0) ?
13199 allocate_small (gen_number, size, acontext, align_const) :
13200 allocate_large (gen_number, size, acontext, align_const));
13204 size_t alloc_context_bytes = acontext->alloc_limit + Align (min_obj_size, align_const) - acontext->alloc_ptr;
13205 int etw_allocation_index = ((gen_number == 0) ? 0 : 1);
13207 etw_allocation_running_amount[etw_allocation_index] += alloc_context_bytes;
13210 if (etw_allocation_running_amount[etw_allocation_index] > etw_allocation_tick)
13212 #ifdef FEATURE_REDHAWK
13213 FIRE_EVENT(GCAllocationTick_V1, (uint32_t)etw_allocation_running_amount[etw_allocation_index],
13214 (gen_number == 0) ? gc_etw_alloc_soh : gc_etw_alloc_loh);
13216 // Unfortunately some of the ETW macros do not check whether the ETW feature is enabled.
13217 // The ones that do are much less efficient.
13218 #if defined(FEATURE_EVENT_TRACE)
13219 if (EVENT_ENABLED(GCAllocationTick_V3))
13221 fire_etw_allocation_event (etw_allocation_running_amount[etw_allocation_index], gen_number, acontext->alloc_ptr);
13223 #endif //FEATURE_EVENT_TRACE
13224 #endif //FEATURE_REDHAWK
13225 etw_allocation_running_amount[etw_allocation_index] = 0;
13229 return (int)can_allocate;
13232 #ifdef MULTIPLE_HEAPS
13233 void gc_heap::balance_heaps (alloc_context* acontext)
13236 if (acontext->alloc_count < 4)
13238 if (acontext->alloc_count == 0)
13240 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, 0) ));
13241 gc_heap* hp = acontext->get_home_heap()->pGenGCHeap;
13242 dprintf (3, ("First allocation for context %Ix on heap %d\n", (size_t)acontext, (size_t)hp->heap_number));
13243 acontext->set_alloc_heap(acontext->get_home_heap());
13244 hp->alloc_context_count++;
13249 BOOL set_home_heap = FALSE;
13252 if (heap_select::can_find_heap_fast())
13254 if (acontext->get_home_heap() != NULL)
13255 hint = acontext->get_home_heap()->pGenGCHeap->heap_number;
13256 if (acontext->get_home_heap() != GCHeap::GetHeap(hint = heap_select::select_heap(acontext, hint)) || ((acontext->alloc_count & 15) == 0))
13258 set_home_heap = TRUE;
13264 if ((acontext->alloc_count & 3) == 0)
13265 set_home_heap = TRUE;
13271 // Since we are balancing up to MAX_SUPPORTED_CPUS, no need for this.
13272 if (n_heaps > MAX_SUPPORTED_CPUS)
13274 // on machines with many processors cache affinity is really king, so don't even try
13275 // to balance on these.
13276 acontext->home_heap = GCHeap::GetHeap( heap_select::select_heap(acontext, hint) );
13277 acontext->alloc_heap = acontext->home_heap;
13282 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13284 dynamic_data* dd = org_hp->dynamic_data_of (0);
13285 ptrdiff_t org_size = dd_new_allocation (dd);
13286 int org_alloc_context_count;
13287 int max_alloc_context_count;
13289 ptrdiff_t max_size;
13290 size_t delta = dd_min_size (dd)/4;
13292 int start, end, finish;
13293 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13294 finish = start + n_heaps;
13300 max_size = org_size + delta;
13301 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, hint) ));
13303 if (org_hp == acontext->get_home_heap()->pGenGCHeap)
13304 max_size = max_size + delta;
13306 org_alloc_context_count = org_hp->alloc_context_count;
13307 max_alloc_context_count = org_alloc_context_count;
13308 if (max_alloc_context_count > 1)
13309 max_size /= max_alloc_context_count;
13311 for (int i = start; i < end; i++)
13313 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13314 dd = hp->dynamic_data_of (0);
13315 ptrdiff_t size = dd_new_allocation (dd);
13316 if (hp == acontext->get_home_heap()->pGenGCHeap)
13317 size = size + delta;
13318 int hp_alloc_context_count = hp->alloc_context_count;
13319 if (hp_alloc_context_count > 0)
13320 size /= (hp_alloc_context_count + 1);
13321 if (size > max_size)
13325 max_alloc_context_count = hp_alloc_context_count;
13329 while (org_alloc_context_count != org_hp->alloc_context_count ||
13330 max_alloc_context_count != max_hp->alloc_context_count);
13332 if ((max_hp == org_hp) && (end < finish))
13334 start = end; end = finish;
13335 delta = dd_min_size(dd)/2; // Make it twice as hard to balance to remote nodes on NUMA.
13339 if (max_hp != org_hp)
13341 org_hp->alloc_context_count--;
13342 max_hp->alloc_context_count++;
13343 acontext->set_alloc_heap(GCHeap::GetHeap(max_hp->heap_number));
13344 if (GCToOSInterface::CanEnableGCCPUGroups())
13345 { //only set ideal processor when max_hp and org_hp are in the same cpu
13346 //group. DO NOT MOVE THREADS ACROSS CPU GROUPS
13347 uint16_t org_gn = heap_select::find_cpu_group_from_heap_no(org_hp->heap_number);
13348 uint16_t max_gn = heap_select::find_cpu_group_from_heap_no(max_hp->heap_number);
13349 if (org_gn == max_gn) //only set within CPU group, so SetThreadIdealProcessor is enough
13351 uint16_t group_proc_no = heap_select::find_group_proc_from_heap_no(max_hp->heap_number);
13353 GCThreadAffinity affinity;
13354 affinity.Processor = group_proc_no;
13355 affinity.Group = org_gn;
13356 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13358 dprintf (3, ("Failed to set the ideal processor and group for heap %d.",
13359 org_hp->heap_number));
13365 uint16_t proc_no = heap_select::find_proc_no_from_heap_no(max_hp->heap_number);
13367 GCThreadAffinity affinity;
13368 affinity.Processor = proc_no;
13369 affinity.Group = GCThreadAffinity::None;
13371 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13373 dprintf (3, ("Failed to set the ideal processor for heap %d.",
13374 org_hp->heap_number));
13377 dprintf (3, ("Switching context %p (home heap %d) ",
13379 acontext->get_home_heap()->pGenGCHeap->heap_number));
13380 dprintf (3, (" from heap %d (%Id free bytes, %d contexts) ",
13381 org_hp->heap_number,
13383 org_alloc_context_count));
13384 dprintf (3, (" to heap %d (%Id free bytes, %d contexts)\n",
13385 max_hp->heap_number,
13386 dd_new_allocation(max_hp->dynamic_data_of(0)),
13387 max_alloc_context_count));
13392 acontext->alloc_count++;
13395 gc_heap* gc_heap::balance_heaps_loh (alloc_context* acontext, size_t /*size*/)
13397 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13398 //dprintf (1, ("LA: %Id", size));
13400 //if (size > 128*1024)
13403 dynamic_data* dd = org_hp->dynamic_data_of (max_generation + 1);
13405 ptrdiff_t org_size = dd_new_allocation (dd);
13407 ptrdiff_t max_size;
13408 size_t delta = dd_min_size (dd) * 4;
13410 int start, end, finish;
13411 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13412 finish = start + n_heaps;
13417 max_size = org_size + delta;
13418 dprintf (3, ("orig hp: %d, max size: %d",
13419 org_hp->heap_number,
13422 for (int i = start; i < end; i++)
13424 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13425 dd = hp->dynamic_data_of (max_generation + 1);
13426 ptrdiff_t size = dd_new_allocation (dd);
13427 dprintf (3, ("hp: %d, size: %d",
13430 if (size > max_size)
13434 dprintf (3, ("max hp: %d, max size: %d",
13435 max_hp->heap_number,
13441 if ((max_hp == org_hp) && (end < finish))
13443 start = end; end = finish;
13444 delta = dd_min_size(dd) * 4; // Need to tuning delta
13448 if (max_hp != org_hp)
13450 dprintf (3, ("loh: %d(%Id)->%d(%Id)",
13451 org_hp->heap_number, dd_new_allocation (org_hp->dynamic_data_of (max_generation + 1)),
13452 max_hp->heap_number, dd_new_allocation (max_hp->dynamic_data_of (max_generation + 1))));
13462 #endif //MULTIPLE_HEAPS
13464 BOOL gc_heap::allocate_more_space(alloc_context* acontext, size_t size,
13465 int alloc_generation_number)
13470 #ifdef MULTIPLE_HEAPS
13471 if (alloc_generation_number == 0)
13473 balance_heaps (acontext);
13474 status = acontext->get_alloc_heap()->pGenGCHeap->try_allocate_more_space (acontext, size, alloc_generation_number);
13478 gc_heap* alloc_heap = balance_heaps_loh (acontext, size);
13479 status = alloc_heap->try_allocate_more_space (acontext, size, alloc_generation_number);
13482 status = try_allocate_more_space (acontext, size, alloc_generation_number);
13483 #endif //MULTIPLE_HEAPS
13485 while (status == -1);
13487 return (status != 0);
13491 CObjectHeader* gc_heap::allocate (size_t jsize, alloc_context* acontext)
13493 size_t size = Align (jsize);
13494 assert (size >= Align (min_obj_size));
13497 uint8_t* result = acontext->alloc_ptr;
13498 acontext->alloc_ptr+=size;
13499 if (acontext->alloc_ptr <= acontext->alloc_limit)
13501 CObjectHeader* obj = (CObjectHeader*)result;
13507 acontext->alloc_ptr -= size;
13510 #pragma inline_depth(0)
13513 if (! allocate_more_space (acontext, size, 0))
13517 #pragma inline_depth(20)
13526 CObjectHeader* gc_heap::try_fast_alloc (size_t jsize)
13528 size_t size = Align (jsize);
13529 assert (size >= Align (min_obj_size));
13530 generation* gen = generation_of (0);
13531 uint8_t* result = generation_allocation_pointer (gen);
13532 generation_allocation_pointer (gen) += size;
13533 if (generation_allocation_pointer (gen) <=
13534 generation_allocation_limit (gen))
13536 return (CObjectHeader*)result;
13540 generation_allocation_pointer (gen) -= size;
13544 void gc_heap::leave_allocation_segment (generation* gen)
13546 adjust_limit (0, 0, gen, max_generation);
13549 void gc_heap::init_free_and_plug()
13551 #ifdef FREE_USAGE_STATS
13552 for (int i = 0; i <= settings.condemned_generation; i++)
13554 generation* gen = generation_of (i);
13555 memset (gen->gen_free_spaces, 0, sizeof (gen->gen_free_spaces));
13556 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13557 memset (gen->gen_current_pinned_free_spaces, 0, sizeof (gen->gen_current_pinned_free_spaces));
13560 if (settings.condemned_generation != max_generation)
13562 for (int i = (settings.condemned_generation + 1); i <= max_generation; i++)
13564 generation* gen = generation_of (i);
13565 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13568 #endif //FREE_USAGE_STATS
13571 void gc_heap::print_free_and_plug (const char* msg)
13573 #if defined(FREE_USAGE_STATS) && defined(SIMPLE_DPRINTF)
13574 int older_gen = ((settings.condemned_generation == max_generation) ? max_generation : (settings.condemned_generation + 1));
13575 for (int i = 0; i <= older_gen; i++)
13577 generation* gen = generation_of (i);
13578 for (int j = 0; j < NUM_GEN_POWER2; j++)
13580 if ((gen->gen_free_spaces[j] != 0) || (gen->gen_plugs[j] != 0))
13582 dprintf (2, ("[%s][h%d][%s#%d]gen%d: 2^%d: F: %Id, P: %Id",
13585 (settings.concurrent ? "BGC" : "GC"),
13588 (j + 9), gen->gen_free_spaces[j], gen->gen_plugs[j]));
13593 UNREFERENCED_PARAMETER(msg);
13594 #endif //FREE_USAGE_STATS && SIMPLE_DPRINTF
13597 void gc_heap::add_gen_plug (int gen_number, size_t plug_size)
13599 #ifdef FREE_USAGE_STATS
13600 dprintf (3, ("adding plug size %Id to gen%d", plug_size, gen_number));
13601 generation* gen = generation_of (gen_number);
13602 size_t sz = BASE_GEN_SIZE;
13605 for (; i < NUM_GEN_POWER2; i++)
13607 if (plug_size < sz)
13614 (gen->gen_plugs[i])++;
13616 UNREFERENCED_PARAMETER(gen_number);
13617 UNREFERENCED_PARAMETER(plug_size);
13618 #endif //FREE_USAGE_STATS
13621 void gc_heap::add_item_to_current_pinned_free (int gen_number, size_t free_size)
13623 #ifdef FREE_USAGE_STATS
13624 generation* gen = generation_of (gen_number);
13625 size_t sz = BASE_GEN_SIZE;
13628 for (; i < NUM_GEN_POWER2; i++)
13630 if (free_size < sz)
13637 (gen->gen_current_pinned_free_spaces[i])++;
13638 generation_pinned_free_obj_space (gen) += free_size;
13639 dprintf (3, ("left pin free %Id(2^%d) to gen%d, total %Id bytes (%Id)",
13640 free_size, (i + 10), gen_number,
13641 generation_pinned_free_obj_space (gen),
13642 gen->gen_current_pinned_free_spaces[i]));
13644 UNREFERENCED_PARAMETER(gen_number);
13645 UNREFERENCED_PARAMETER(free_size);
13646 #endif //FREE_USAGE_STATS
13649 void gc_heap::add_gen_free (int gen_number, size_t free_size)
13651 #ifdef FREE_USAGE_STATS
13652 dprintf (3, ("adding free size %Id to gen%d", free_size, gen_number));
13653 generation* gen = generation_of (gen_number);
13654 size_t sz = BASE_GEN_SIZE;
13657 for (; i < NUM_GEN_POWER2; i++)
13659 if (free_size < sz)
13666 (gen->gen_free_spaces[i])++;
13668 UNREFERENCED_PARAMETER(gen_number);
13669 UNREFERENCED_PARAMETER(free_size);
13670 #endif //FREE_USAGE_STATS
13673 void gc_heap::remove_gen_free (int gen_number, size_t free_size)
13675 #ifdef FREE_USAGE_STATS
13676 dprintf (3, ("removing free %Id from gen%d", free_size, gen_number));
13677 generation* gen = generation_of (gen_number);
13678 size_t sz = BASE_GEN_SIZE;
13681 for (; i < NUM_GEN_POWER2; i++)
13683 if (free_size < sz)
13690 (gen->gen_free_spaces[i])--;
13692 UNREFERENCED_PARAMETER(gen_number);
13693 UNREFERENCED_PARAMETER(free_size);
13694 #endif //FREE_USAGE_STATS
13697 uint8_t* gc_heap::allocate_in_older_generation (generation* gen, size_t size,
13698 int from_gen_number,
13699 uint8_t* old_loc REQD_ALIGN_AND_OFFSET_DCL)
13701 size = Align (size);
13702 assert (size >= Align (min_obj_size));
13703 assert (from_gen_number < max_generation);
13704 assert (from_gen_number >= 0);
13705 assert (generation_of (from_gen_number + 1) == gen);
13707 allocator* gen_allocator = generation_allocator (gen);
13708 BOOL discard_p = gen_allocator->discard_if_no_fit_p ();
13709 int pad_in_front = (old_loc != 0)? USE_PADDING_FRONT : 0;
13711 size_t real_size = size + Align (min_obj_size);
13713 real_size += Align (min_obj_size);
13715 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
13716 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front)))
13718 size_t sz_list = gen_allocator->first_bucket_size();
13719 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
13721 if ((real_size < (sz_list / 2)) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
13723 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
13724 uint8_t* prev_free_item = 0;
13725 while (free_list != 0)
13727 dprintf (3, ("considering free list %Ix", (size_t)free_list));
13729 size_t free_list_size = unused_array_size (free_list);
13731 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + free_list_size),
13732 old_loc, USE_PADDING_TAIL | pad_in_front))
13734 dprintf (4, ("F:%Ix-%Id",
13735 (size_t)free_list, free_list_size));
13737 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, !discard_p);
13738 generation_free_list_space (gen) -= free_list_size;
13739 remove_gen_free (gen->gen_num, free_list_size);
13741 adjust_limit (free_list, free_list_size, gen, from_gen_number+1);
13744 // We do first fit on bucket 0 because we are not guaranteed to find a fit there.
13745 else if (discard_p || (a_l_idx == 0))
13747 dprintf (3, ("couldn't use this free area, discarding"));
13748 generation_free_obj_space (gen) += free_list_size;
13750 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
13751 generation_free_list_space (gen) -= free_list_size;
13752 remove_gen_free (gen->gen_num, free_list_size);
13756 prev_free_item = free_list;
13758 free_list = free_list_slot (free_list);
13761 sz_list = sz_list * 2;
13763 //go back to the beginning of the segment list
13764 generation_allocate_end_seg_p (gen) = TRUE;
13765 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
13766 if (seg != generation_allocation_segment (gen))
13768 leave_allocation_segment (gen);
13769 generation_allocation_segment (gen) = seg;
13771 while (seg != ephemeral_heap_segment)
13773 if (size_fit_p(size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
13774 heap_segment_committed (seg), old_loc, USE_PADDING_TAIL | pad_in_front))
13776 dprintf (3, ("using what's left in committed"));
13777 adjust_limit (heap_segment_plan_allocated (seg),
13778 heap_segment_committed (seg) -
13779 heap_segment_plan_allocated (seg),
13780 gen, from_gen_number+1);
13781 // dformat (t, 3, "Expanding segment allocation");
13782 heap_segment_plan_allocated (seg) =
13783 heap_segment_committed (seg);
13788 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
13789 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
13790 grow_heap_segment (seg, heap_segment_plan_allocated (seg), old_loc, size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG))
13792 dprintf (3, ("using what's left in reserved"));
13793 adjust_limit (heap_segment_plan_allocated (seg),
13794 heap_segment_committed (seg) -
13795 heap_segment_plan_allocated (seg),
13796 gen, from_gen_number+1);
13797 heap_segment_plan_allocated (seg) =
13798 heap_segment_committed (seg);
13804 leave_allocation_segment (gen);
13805 heap_segment* next_seg = heap_segment_next_rw (seg);
13808 dprintf (3, ("getting next segment"));
13809 generation_allocation_segment (gen) = next_seg;
13810 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
13811 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
13820 seg = generation_allocation_segment (gen);
13822 //No need to fix the last region. Will be done later
13833 uint8_t* result = generation_allocation_pointer (gen);
13837 if ((pad_in_front & USE_PADDING_FRONT) &&
13838 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
13839 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
13841 pad = Align (min_obj_size);
13842 set_plug_padded (old_loc);
13844 #endif //SHORT_PLUGS
13846 #ifdef FEATURE_STRUCTALIGN
13847 _ASSERTE(!old_loc || alignmentOffset != 0);
13848 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
13851 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
13852 set_node_aligninfo (old_loc, requiredAlignment, pad1);
13855 #else // FEATURE_STRUCTALIGN
13856 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
13858 pad += switch_alignment_size (is_plug_padded (old_loc));
13859 set_node_realigned (old_loc);
13860 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
13861 (size_t)old_loc, (size_t)(result+pad)));
13862 assert (same_large_alignment_p (result + pad, old_loc));
13864 #endif // FEATURE_STRUCTALIGN
13865 dprintf (3, ("Allocate %Id bytes", size));
13867 if ((old_loc == 0) || (pad != 0))
13869 //allocating a non plug or a gap, so reset the start region
13870 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
13873 generation_allocation_pointer (gen) += size + pad;
13874 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
13875 if (generation_allocate_end_seg_p (gen))
13877 generation_end_seg_allocated (gen) += size;
13881 generation_free_list_allocated (gen) += size;
13883 generation_allocation_size (gen) += size;
13885 dprintf (3, ("aio: ptr: %Ix, limit: %Ix, sr: %Ix",
13886 generation_allocation_pointer (gen), generation_allocation_limit (gen),
13887 generation_allocation_context_start_region (gen)));
13889 return result + pad;;
13893 void gc_heap::repair_allocation_in_expanded_heap (generation* consing_gen)
13895 //make sure that every generation has a planned allocation start
13896 int gen_number = max_generation - 1;
13897 while (gen_number>= 0)
13899 generation* gen = generation_of (gen_number);
13900 if (0 == generation_plan_allocation_start (gen))
13902 realloc_plan_generation_start (gen, consing_gen);
13904 assert (generation_plan_allocation_start (gen));
13909 // now we know the planned allocation size
13910 size_t size = (generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
13911 heap_segment* seg = generation_allocation_segment (consing_gen);
13912 if (generation_allocation_limit (consing_gen) == heap_segment_plan_allocated (seg))
13916 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
13921 assert (settings.condemned_generation == max_generation);
13922 uint8_t* first_address = generation_allocation_limit (consing_gen);
13923 //look through the pinned plugs for relevant ones.
13924 //Look for the right pinned plug to start from.
13927 while (mi != mark_stack_tos)
13929 m = pinned_plug_of (mi);
13930 if ((pinned_plug (m) == first_address))
13935 assert (mi != mark_stack_tos);
13936 pinned_len (m) = size;
13940 //tododefrag optimize for new segment (plan_allocated == mem)
13941 uint8_t* gc_heap::allocate_in_expanded_heap (generation* gen,
13946 BOOL set_padding_on_saved_p,
13947 mark* pinned_plug_entry,
13948 #endif //SHORT_PLUGS
13949 BOOL consider_bestfit,
13950 int active_new_gen_number
13951 REQD_ALIGN_AND_OFFSET_DCL)
13953 UNREFERENCED_PARAMETER(active_new_gen_number);
13954 dprintf (3, ("aie: P: %Ix, size: %Ix", old_loc, size));
13956 size = Align (size);
13957 assert (size >= Align (min_obj_size));
13958 int pad_in_front = (old_loc != 0) ? USE_PADDING_FRONT : 0;
13960 if (consider_bestfit && use_bestfit)
13962 assert (bestfit_seg);
13963 dprintf (SEG_REUSE_LOG_1, ("reallocating 0x%Ix in expanded heap, size: %Id",
13965 return bestfit_seg->fit (old_loc,
13967 set_padding_on_saved_p,
13969 #endif //SHORT_PLUGS
13970 size REQD_ALIGN_AND_OFFSET_ARG);
13973 heap_segment* seg = generation_allocation_segment (gen);
13975 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
13976 generation_allocation_limit (gen), old_loc,
13977 ((generation_allocation_limit (gen) !=
13978 heap_segment_plan_allocated (seg))? USE_PADDING_TAIL : 0) | pad_in_front)))
13980 dprintf (3, ("aie: can't fit: ptr: %Ix, limit: %Ix", generation_allocation_pointer (gen),
13981 generation_allocation_limit (gen)));
13984 uint8_t* first_address = (generation_allocation_limit (gen) ?
13985 generation_allocation_limit (gen) :
13986 heap_segment_mem (seg));
13987 assert (in_range_for_segment (first_address, seg));
13989 uint8_t* end_address = heap_segment_reserved (seg);
13991 dprintf (3, ("aie: first_addr: %Ix, gen alloc limit: %Ix, end_address: %Ix",
13992 first_address, generation_allocation_limit (gen), end_address));
13997 if (heap_segment_allocated (seg) != heap_segment_mem (seg))
13999 assert (settings.condemned_generation == max_generation);
14000 //look through the pinned plugs for relevant ones.
14001 //Look for the right pinned plug to start from.
14002 while (mi != mark_stack_tos)
14004 m = pinned_plug_of (mi);
14005 if ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address))
14007 dprintf (3, ("aie: found pin: %Ix", pinned_plug (m)));
14013 if (mi != mark_stack_tos)
14015 //fix old free list.
14016 size_t hsize = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
14018 dprintf(3,("gc filling up hole"));
14019 ptrdiff_t mi1 = (ptrdiff_t)mi;
14020 while ((mi1 >= 0) &&
14021 (pinned_plug (pinned_plug_of(mi1)) != generation_allocation_limit (gen)))
14023 dprintf (3, ("aie: checking pin %Ix", pinned_plug (pinned_plug_of(mi1))));
14028 size_t saved_pinned_len = pinned_len (pinned_plug_of(mi1));
14029 pinned_len (pinned_plug_of(mi1)) = hsize;
14030 dprintf (3, ("changing %Ix len %Ix->%Ix",
14031 pinned_plug (pinned_plug_of(mi1)),
14032 saved_pinned_len, pinned_len (pinned_plug_of(mi1))));
14039 assert (generation_allocation_limit (gen) ==
14040 generation_allocation_pointer (gen));
14041 mi = mark_stack_tos;
14044 while ((mi != mark_stack_tos) && in_range_for_segment (pinned_plug (m), seg))
14046 size_t len = pinned_len (m);
14047 uint8_t* free_list = (pinned_plug (m) - len);
14048 dprintf (3, ("aie: testing free item: %Ix->%Ix(%Ix)",
14049 free_list, (free_list + len), len));
14050 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + len), old_loc, USE_PADDING_TAIL | pad_in_front))
14052 dprintf (3, ("aie: Found adequate unused area: %Ix, size: %Id",
14053 (size_t)free_list, len));
14055 generation_allocation_pointer (gen) = free_list;
14056 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14057 generation_allocation_limit (gen) = (free_list + len);
14059 goto allocate_in_free;
14062 m = pinned_plug_of (mi);
14065 //switch to the end of the segment.
14066 generation_allocation_pointer (gen) = heap_segment_plan_allocated (seg);
14067 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14068 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14069 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14070 dprintf (3, ("aie: switching to end of seg: %Ix->%Ix(%Ix)",
14071 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14072 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
14074 if (!size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14075 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front))
14077 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, can't alloc", generation_allocation_pointer (gen),
14078 generation_allocation_limit (gen)));
14079 assert (!"Can't allocate if no free space");
14090 uint8_t* result = generation_allocation_pointer (gen);
14094 if ((pad_in_front & USE_PADDING_FRONT) &&
14095 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14096 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14099 pad = Align (min_obj_size);
14100 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
14102 #endif //SHORT_PLUGS
14104 #ifdef FEATURE_STRUCTALIGN
14105 _ASSERTE(!old_loc || alignmentOffset != 0);
14106 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14109 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14110 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14114 #else // FEATURE_STRUCTALIGN
14115 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14117 pad += switch_alignment_size (is_plug_padded (old_loc));
14118 set_node_realigned (old_loc);
14119 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14120 (size_t)old_loc, (size_t)(result+pad)));
14121 assert (same_large_alignment_p (result + pad, old_loc));
14124 #endif // FEATURE_STRUCTALIGN
14126 if ((old_loc == 0) || (pad != 0))
14128 //allocating a non plug or a gap, so reset the start region
14129 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14132 generation_allocation_pointer (gen) += size + pad;
14133 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14134 dprintf (3, ("Allocated in expanded heap %Ix:%Id", (size_t)(result+pad), size));
14136 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, sr: %Ix",
14137 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14138 generation_allocation_context_start_region (gen)));
14140 return result + pad;
14144 generation* gc_heap::ensure_ephemeral_heap_segment (generation* consing_gen)
14146 heap_segment* seg = generation_allocation_segment (consing_gen);
14147 if (seg != ephemeral_heap_segment)
14149 assert (generation_allocation_pointer (consing_gen)>= heap_segment_mem (seg));
14150 assert (generation_allocation_pointer (consing_gen)<= heap_segment_committed (seg));
14152 //fix the allocated size of the segment.
14153 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
14155 generation* new_consing_gen = generation_of (max_generation - 1);
14156 generation_allocation_pointer (new_consing_gen) =
14157 heap_segment_mem (ephemeral_heap_segment);
14158 generation_allocation_limit (new_consing_gen) =
14159 generation_allocation_pointer (new_consing_gen);
14160 generation_allocation_context_start_region (new_consing_gen) =
14161 generation_allocation_pointer (new_consing_gen);
14162 generation_allocation_segment (new_consing_gen) = ephemeral_heap_segment;
14164 return new_consing_gen;
14167 return consing_gen;
14170 uint8_t* gc_heap::allocate_in_condemned_generations (generation* gen,
14172 int from_gen_number,
14174 BOOL* convert_to_pinned_p,
14175 uint8_t* next_pinned_plug,
14176 heap_segment* current_seg,
14177 #endif //SHORT_PLUGS
14179 REQD_ALIGN_AND_OFFSET_DCL)
14181 // Make sure that the youngest generation gap hasn't been allocated
14182 if (settings.promotion)
14184 assert (generation_plan_allocation_start (youngest_generation) == 0);
14187 size = Align (size);
14188 assert (size >= Align (min_obj_size));
14189 int to_gen_number = from_gen_number;
14190 if (from_gen_number != (int)max_generation)
14192 to_gen_number = from_gen_number + (settings.promotion ? 1 : 0);
14195 dprintf (3, ("aic gen%d: s: %Id, %d->%d, %Ix->%Ix", gen->gen_num, size, from_gen_number,
14196 to_gen_number, generation_allocation_pointer(gen), generation_allocation_limit(gen)));
14198 int pad_in_front = (old_loc != 0) ? USE_PADDING_FRONT : 0;
14200 if ((from_gen_number != -1) && (from_gen_number != (int)max_generation) && settings.promotion)
14202 generation_condemned_allocated (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14203 generation_allocation_size (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14207 heap_segment* seg = generation_allocation_segment (gen);
14208 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14209 generation_allocation_limit (gen), old_loc,
14210 ((generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))?USE_PADDING_TAIL:0)|pad_in_front)))
14212 if ((! (pinned_plug_que_empty_p()) &&
14213 (generation_allocation_limit (gen) ==
14214 pinned_plug (oldest_pin()))))
14216 size_t entry = deque_pinned_plug();
14217 mark* pinned_plug_entry = pinned_plug_of (entry);
14218 size_t len = pinned_len (pinned_plug_entry);
14219 uint8_t* plug = pinned_plug (pinned_plug_entry);
14220 set_new_pin_info (pinned_plug_entry, generation_allocation_pointer (gen));
14222 #ifdef FREE_USAGE_STATS
14223 generation_allocated_in_pinned_free (gen) += generation_allocated_since_last_pin (gen);
14224 dprintf (3, ("allocated %Id so far within pin %Ix, total->%Id",
14225 generation_allocated_since_last_pin (gen),
14227 generation_allocated_in_pinned_free (gen)));
14228 generation_allocated_since_last_pin (gen) = 0;
14230 add_item_to_current_pinned_free (gen->gen_num, pinned_len (pinned_plug_of (entry)));
14231 #endif //FREE_USAGE_STATS
14233 dprintf (3, ("mark stack bos: %Id, tos: %Id, aic: p %Ix len: %Ix->%Ix",
14234 mark_stack_bos, mark_stack_tos, plug, len, pinned_len (pinned_plug_of (entry))));
14236 assert(mark_stack_array[entry].len == 0 ||
14237 mark_stack_array[entry].len >= Align(min_obj_size));
14238 generation_allocation_pointer (gen) = plug + len;
14239 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14240 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14241 set_allocator_next_pin (gen);
14243 //Add the size of the pinned plug to the right pinned allocations
14244 //find out which gen this pinned plug came from
14245 int frgn = object_gennum (plug);
14246 if ((frgn != (int)max_generation) && settings.promotion)
14248 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
14249 int togn = object_gennum_plan (plug);
14252 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
14258 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
14260 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14261 dprintf (3, ("changed limit to plan alloc: %Ix", generation_allocation_limit (gen)));
14265 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
14267 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14268 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14269 dprintf (3, ("changed limit to commit: %Ix", generation_allocation_limit (gen)));
14273 #ifndef RESPECT_LARGE_ALIGNMENT
14274 assert (gen != youngest_generation);
14275 #endif //RESPECT_LARGE_ALIGNMENT
14277 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14278 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
14279 (grow_heap_segment (seg, generation_allocation_pointer (gen), old_loc,
14280 size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG)))
14282 dprintf (3, ("Expanded segment allocation by committing more memory"));
14283 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14284 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14288 heap_segment* next_seg = heap_segment_next (seg);
14289 assert (generation_allocation_pointer (gen)>=
14290 heap_segment_mem (seg));
14291 // Verify that all pinned plugs for this segment are consumed
14292 if (!pinned_plug_que_empty_p() &&
14293 ((pinned_plug (oldest_pin()) <
14294 heap_segment_allocated (seg)) &&
14295 (pinned_plug (oldest_pin()) >=
14296 generation_allocation_pointer (gen))))
14298 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
14299 pinned_plug (oldest_pin())));
14302 assert (generation_allocation_pointer (gen)>=
14303 heap_segment_mem (seg));
14304 assert (generation_allocation_pointer (gen)<=
14305 heap_segment_committed (seg));
14306 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
14310 generation_allocation_segment (gen) = next_seg;
14311 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
14312 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
14313 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14317 return 0; //should only happen during allocation of generation 0 gap
14318 // in that case we are going to grow the heap anyway
14323 set_allocator_next_pin (gen);
14330 assert (generation_allocation_pointer (gen)>=
14331 heap_segment_mem (generation_allocation_segment (gen)));
14332 uint8_t* result = generation_allocation_pointer (gen);
14335 if ((pad_in_front & USE_PADDING_FRONT) &&
14336 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14337 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14339 ptrdiff_t dist = old_loc - result;
14342 dprintf (3, ("old alloc: %Ix, same as new alloc, not padding", old_loc));
14347 if ((dist > 0) && (dist < (ptrdiff_t)Align (min_obj_size)))
14349 dprintf (3, ("old alloc: %Ix, only %d bytes > new alloc! Shouldn't happen", old_loc, dist));
14353 pad = Align (min_obj_size);
14354 set_plug_padded (old_loc);
14357 #endif //SHORT_PLUGS
14358 #ifdef FEATURE_STRUCTALIGN
14359 _ASSERTE(!old_loc || alignmentOffset != 0);
14360 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14361 if ((old_loc != 0))
14363 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14364 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14367 #else // FEATURE_STRUCTALIGN
14368 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14370 pad += switch_alignment_size (is_plug_padded (old_loc));
14371 set_node_realigned(old_loc);
14372 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14373 (size_t)old_loc, (size_t)(result+pad)));
14374 assert (same_large_alignment_p (result + pad, old_loc));
14376 #endif // FEATURE_STRUCTALIGN
14379 if ((next_pinned_plug != 0) && (pad != 0) && (generation_allocation_segment (gen) == current_seg))
14381 assert (old_loc != 0);
14382 ptrdiff_t dist_to_next_pin = (ptrdiff_t)(next_pinned_plug - (generation_allocation_pointer (gen) + size + pad));
14383 assert (dist_to_next_pin >= 0);
14385 if ((dist_to_next_pin >= 0) && (dist_to_next_pin < (ptrdiff_t)Align (min_obj_size)))
14387 dprintf (3, ("%Ix->(%Ix,%Ix),%Ix(%Ix)(%Ix),NP->PP",
14389 generation_allocation_pointer (gen),
14390 generation_allocation_limit (gen),
14393 dist_to_next_pin));
14394 clear_plug_padded (old_loc);
14396 *convert_to_pinned_p = TRUE;
14397 record_interesting_data_point (idp_converted_pin);
14402 #endif //SHORT_PLUGS
14404 if ((old_loc == 0) || (pad != 0))
14406 //allocating a non plug or a gap, so reset the start region
14407 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14410 generation_allocation_pointer (gen) += size + pad;
14411 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14413 #ifdef FREE_USAGE_STATS
14414 generation_allocated_since_last_pin (gen) += size;
14415 #endif //FREE_USAGE_STATS
14417 dprintf (3, ("aic: ptr: %Ix, limit: %Ix, sr: %Ix",
14418 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14419 generation_allocation_context_start_region (gen)));
14421 assert (result + pad);
14422 return result + pad;
14426 inline int power (int x, int y)
14429 for (int i = 0; i < y; i++)
14436 int gc_heap::joined_generation_to_condemn (BOOL should_evaluate_elevation,
14438 BOOL* blocking_collection_p
14439 STRESS_HEAP_ARG(int n_original))
14442 #ifdef MULTIPLE_HEAPS
14443 BOOL blocking_p = *blocking_collection_p;
14446 for (int i = 0; i < n_heaps; i++)
14448 if (g_heaps[i]->last_gc_before_oom)
14450 dprintf (GTC_LOG, ("h%d is setting blocking to TRUE", i));
14451 *blocking_collection_p = TRUE;
14456 #endif //MULTIPLE_HEAPS
14458 if (should_evaluate_elevation && (n == max_generation))
14460 dprintf (GTC_LOG, ("lock: %d(%d)",
14461 (settings.should_lock_elevation ? 1 : 0),
14462 settings.elevation_locked_count));
14464 if (settings.should_lock_elevation)
14466 settings.elevation_locked_count++;
14467 if (settings.elevation_locked_count == 6)
14469 settings.elevation_locked_count = 0;
14473 n = max_generation - 1;
14474 settings.elevation_reduced = TRUE;
14479 settings.elevation_locked_count = 0;
14484 settings.should_lock_elevation = FALSE;
14485 settings.elevation_locked_count = 0;
14489 #ifdef BACKGROUND_GC
14490 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
14491 // generations to be collected,
14493 // [LOCALGC TODO] STRESS_HEAP is not defined for a standalone GC so there are multiple
14494 // things that need to be fixed in this code block.
14495 if (n_original != max_generation &&
14496 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
14498 #ifndef FEATURE_REDHAWK
14499 // for the GC stress mix mode throttle down gen2 collections
14500 if (g_pConfig->IsGCStressMix())
14502 size_t current_gc_count = 0;
14504 #ifdef MULTIPLE_HEAPS
14505 current_gc_count = (size_t)dd_collection_count (g_heaps[0]->dynamic_data_of (0));
14507 current_gc_count = (size_t)dd_collection_count (dynamic_data_of (0));
14508 #endif //MULTIPLE_HEAPS
14509 // in gc stress, only escalate every 10th non-gen2 collection to a gen2...
14510 if ((current_gc_count % 10) == 0)
14512 n = max_generation;
14515 // for traditional GC stress
14517 #endif // !FEATURE_REDHAWK
14518 if (*blocking_collection_p)
14520 // We call StressHeap() a lot for Concurrent GC Stress. However,
14521 // if we can not do a concurrent collection, no need to stress anymore.
14522 // @TODO: Enable stress when the memory pressure goes down again
14523 GCStressPolicy::GlobalDisable();
14527 n = max_generation;
14530 #endif //BACKGROUND_GC
14531 #endif //STRESS_HEAP
14537 size_t get_survived_size (gc_history_per_heap* hist)
14539 size_t surv_size = 0;
14540 gc_generation_data* gen_data;
14542 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
14544 gen_data = &(hist->gen_data[gen_number]);
14545 surv_size += (gen_data->size_after -
14546 gen_data->free_list_space_after -
14547 gen_data->free_obj_space_after);
14553 size_t gc_heap::get_total_survived_size()
14555 size_t total_surv_size = 0;
14556 #ifdef MULTIPLE_HEAPS
14557 for (int i = 0; i < gc_heap::n_heaps; i++)
14559 gc_heap* hp = gc_heap::g_heaps[i];
14560 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
14561 total_surv_size += get_survived_size (current_gc_data_per_heap);
14564 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
14565 total_surv_size = get_survived_size (current_gc_data_per_heap);
14566 #endif //MULTIPLE_HEAPS
14567 return total_surv_size;
14570 // Gets what's allocated on both SOH and LOH that hasn't been collected.
14571 size_t gc_heap::get_current_allocated()
14573 dynamic_data* dd = dynamic_data_of (0);
14574 size_t current_alloc = dd_desired_allocation (dd) - dd_new_allocation (dd);
14575 dd = dynamic_data_of (max_generation + 1);
14576 current_alloc += dd_desired_allocation (dd) - dd_new_allocation (dd);
14578 return current_alloc;
14581 size_t gc_heap::get_total_allocated()
14583 size_t total_current_allocated = 0;
14584 #ifdef MULTIPLE_HEAPS
14585 for (int i = 0; i < gc_heap::n_heaps; i++)
14587 gc_heap* hp = gc_heap::g_heaps[i];
14588 total_current_allocated += hp->get_current_allocated();
14591 total_current_allocated = get_current_allocated();
14592 #endif //MULTIPLE_HEAPS
14593 return total_current_allocated;
14596 size_t gc_heap::current_generation_size (int gen_number)
14598 dynamic_data* dd = dynamic_data_of (gen_number);
14599 size_t gen_size = (dd_current_size (dd) + dd_desired_allocation (dd)
14600 - dd_new_allocation (dd));
14606 #pragma warning(push)
14607 #pragma warning(disable:6326) // "Potential comparison of a constant with another constant" is intentional in this function.
14611 This is called by when we are actually doing a GC, or when we are just checking whether
14612 we would do a full blocking GC, in which case check_only_p is TRUE.
14614 The difference between calling this with check_only_p TRUE and FALSE is that when it's
14616 settings.reason is ignored
14617 budgets are not checked (since they are checked before this is called)
14618 it doesn't change anything non local like generation_skip_ratio
14620 int gc_heap::generation_to_condemn (int n_initial,
14621 BOOL* blocking_collection_p,
14622 BOOL* elevation_requested_p,
14625 gc_mechanisms temp_settings = settings;
14626 gen_to_condemn_tuning temp_condemn_reasons;
14627 gc_mechanisms* local_settings = (check_only_p ? &temp_settings : &settings);
14628 gen_to_condemn_tuning* local_condemn_reasons = (check_only_p ? &temp_condemn_reasons : &gen_to_condemn_reasons);
14631 if ((local_settings->reason == reason_oos_soh) || (local_settings->reason == reason_oos_loh))
14633 assert (n_initial >= 1);
14636 assert (settings.reason != reason_empty);
14639 local_condemn_reasons->init();
14643 if (heap_number == 0)
14645 dprintf (GTC_LOG, ("init: %d(%d)", n_initial, settings.reason));
14649 BOOL low_memory_detected = g_low_memory_status;
14650 uint32_t memory_load = 0;
14651 uint64_t available_physical = 0;
14652 uint64_t available_page_file = 0;
14653 BOOL check_memory = FALSE;
14654 BOOL high_fragmentation = FALSE;
14655 BOOL v_high_memory_load = FALSE;
14656 BOOL high_memory_load = FALSE;
14657 BOOL low_ephemeral_space = FALSE;
14658 BOOL evaluate_elevation = TRUE;
14659 *elevation_requested_p = FALSE;
14660 *blocking_collection_p = FALSE;
14662 BOOL check_max_gen_alloc = TRUE;
14666 #endif //STRESS_HEAP
14670 dd_fragmentation (dynamic_data_of (0)) =
14671 generation_free_list_space (youngest_generation) +
14672 generation_free_obj_space (youngest_generation);
14674 dd_fragmentation (dynamic_data_of (max_generation + 1)) =
14675 generation_free_list_space (large_object_generation) +
14676 generation_free_obj_space (large_object_generation);
14678 //save new_allocation
14679 for (i = 0; i <= max_generation+1; i++)
14681 dynamic_data* dd = dynamic_data_of (i);
14682 dprintf (GTC_LOG, ("h%d: g%d: l: %Id (%Id)",
14684 dd_new_allocation (dd),
14685 dd_desired_allocation (dd)));
14686 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
14689 local_condemn_reasons->set_gen (gen_initial, n);
14692 #ifdef BACKGROUND_GC
14693 if (recursive_gc_sync::background_running_p())
14695 dprintf (GTC_LOG, ("bgc in prog, 1"));
14696 check_max_gen_alloc = FALSE;
14698 #endif //BACKGROUND_GC
14700 if (check_max_gen_alloc)
14702 //figure out if large objects need to be collected.
14703 if (get_new_allocation (max_generation+1) <= 0)
14705 n = max_generation;
14706 local_condemn_reasons->set_gen (gen_alloc_budget, n);
14710 //figure out which generation ran out of allocation
14711 for (i = n+1; i <= (check_max_gen_alloc ? max_generation : (max_generation - 1)); i++)
14713 if (get_new_allocation (i) <= 0)
14724 local_condemn_reasons->set_gen (gen_alloc_budget, n);
14727 dprintf (GTC_LOG, ("h%d: g%d budget", heap_number, ((get_new_allocation (max_generation+1) <= 0) ? 3 : n)));
14731 #if defined(BACKGROUND_GC) && !defined(MULTIPLE_HEAPS)
14732 //time based tuning
14733 // if enough time has elapsed since the last gc
14734 // and the number of gc is too low (1/10 of lower gen) then collect
14735 // This should also be enabled if we have memory concerns
14736 int n_time_max = max_generation;
14740 if (recursive_gc_sync::background_running_p())
14742 n_time_max = max_generation - 1;
14746 if ((local_settings->pause_mode == pause_interactive) ||
14747 (local_settings->pause_mode == pause_sustained_low_latency))
14749 dynamic_data* dd0 = dynamic_data_of (0);
14750 size_t now = GetHighPrecisionTimeStamp();
14752 for (i = (temp_gen+1); i <= n_time_max; i++)
14754 dynamic_data* dd = dynamic_data_of (i);
14755 if ((now > dd_time_clock(dd) + dd_time_clock_interval(dd)) &&
14756 (dd_gc_clock (dd0) > (dd_gc_clock (dd) + dd_gc_clock_interval(dd))) &&
14757 ((n < max_generation) || ((dd_current_size (dd) < dd_max_size (dd0)))))
14759 n = min (i, n_time_max);
14760 dprintf (GTC_LOG, ("time %d", n));
14765 local_condemn_reasons->set_gen (gen_time_tuning, n);
14771 dprintf (GTC_LOG, ("Condemning %d based on time tuning and fragmentation", n));
14773 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
14775 if (n < (max_generation - 1))
14777 if (dt_low_card_table_efficiency_p (tuning_deciding_condemned_gen))
14779 n = max (n, max_generation - 1);
14780 local_settings->promotion = TRUE;
14781 dprintf (GTC_LOG, ("h%d: skip %d, c %d",
14782 heap_number, generation_skip_ratio, n));
14783 local_condemn_reasons->set_condition (gen_low_card_p);
14789 generation_skip_ratio = 100;
14792 if (dt_low_ephemeral_space_p (check_only_p ?
14793 tuning_deciding_full_gc :
14794 tuning_deciding_condemned_gen))
14796 low_ephemeral_space = TRUE;
14798 n = max (n, max_generation - 1);
14799 local_condemn_reasons->set_condition (gen_low_ephemeral_p);
14800 dprintf (GTC_LOG, ("h%d: low eph", heap_number));
14802 #ifdef BACKGROUND_GC
14803 if (!gc_can_use_concurrent || (generation_free_list_space (generation_of (max_generation)) == 0))
14804 #endif //BACKGROUND_GC
14806 //It is better to defragment first if we are running out of space for
14807 //the ephemeral generation but we have enough fragmentation to make up for it
14808 //in the non ephemeral generation. Essentially we are trading a gen2 for
14809 // having to expand heap in ephemeral collections.
14810 if (dt_high_frag_p (tuning_deciding_condemned_gen,
14811 max_generation - 1,
14814 high_fragmentation = TRUE;
14815 local_condemn_reasons->set_condition (gen_max_high_frag_e_p);
14816 dprintf (GTC_LOG, ("heap%d: gen1 frag", heap_number));
14821 //figure out which ephemeral generation is too fragramented
14823 for (i = n+1; i < max_generation; i++)
14825 if (dt_high_frag_p (tuning_deciding_condemned_gen, i))
14827 dprintf (GTC_LOG, ("h%d g%d too frag", heap_number, i));
14834 if (low_ephemeral_space)
14837 local_settings->promotion = TRUE;
14842 local_condemn_reasons->set_condition (gen_eph_high_frag_p);
14847 if (settings.pause_mode == pause_low_latency)
14849 if (!is_induced (settings.reason))
14851 n = min (n, max_generation - 1);
14852 dprintf (GTC_LOG, ("low latency mode is enabled, condemning %d", n));
14853 evaluate_elevation = FALSE;
14859 // It's hard to catch when we get to the point that the memory load is so high
14860 // we get an induced GC from the finalizer thread so we are checking the memory load
14861 // for every gen0 GC.
14862 check_memory = (check_only_p ?
14864 ((n >= 1) || low_memory_detected));
14868 //find out if we are short on memory
14869 get_memory_info (&memory_load, &available_physical, &available_page_file);
14870 if (heap_number == 0)
14872 dprintf (GTC_LOG, ("ml: %d", memory_load));
14875 // Need to get it early enough for all heaps to use.
14876 entry_available_physical_mem = available_physical;
14877 local_settings->entry_memory_load = memory_load;
14879 // @TODO: Force compaction more often under GCSTRESS
14880 if (memory_load >= high_memory_load_th || low_memory_detected)
14882 #ifdef SIMPLE_DPRINTF
14883 // stress log can't handle any parameter that's bigger than a void*.
14884 if (heap_number == 0)
14886 dprintf (GTC_LOG, ("tp: %I64d, ap: %I64d", total_physical_mem, available_physical));
14888 #endif //SIMPLE_DPRINTF
14890 high_memory_load = TRUE;
14892 if (memory_load >= v_high_memory_load_th || low_memory_detected)
14894 // TODO: Perhaps in 64-bit we should be estimating gen1's fragmentation as well since
14895 // gen1/gen0 may take a lot more memory than gen2.
14896 if (!high_fragmentation)
14898 high_fragmentation = dt_estimate_reclaim_space_p (tuning_deciding_condemned_gen, max_generation);
14900 v_high_memory_load = TRUE;
14904 if (!high_fragmentation)
14906 high_fragmentation = dt_estimate_high_frag_p (tuning_deciding_condemned_gen, max_generation, available_physical);
14910 if (high_fragmentation)
14912 if (high_memory_load)
14914 local_condemn_reasons->set_condition (gen_max_high_frag_m_p);
14916 else if (v_high_memory_load)
14918 local_condemn_reasons->set_condition (gen_max_high_frag_vm_p);
14924 dprintf (GTC_LOG, ("h%d: le: %d, hm: %d, vm: %d, f: %d",
14925 heap_number, low_ephemeral_space, high_memory_load, v_high_memory_load,
14926 high_fragmentation));
14928 if (should_expand_in_full_gc)
14930 dprintf (GTC_LOG, ("h%d: expand_in_full - BLOCK", heap_number));
14931 *blocking_collection_p = TRUE;
14934 should_expand_in_full_gc = FALSE;
14936 evaluate_elevation = FALSE;
14937 n = max_generation;
14938 local_condemn_reasons->set_condition (gen_expand_fullgc_p);
14941 if (last_gc_before_oom)
14943 dprintf (GTC_LOG, ("h%d: alloc full - BLOCK", heap_number));
14944 n = max_generation;
14945 *blocking_collection_p = TRUE;
14946 if ((local_settings->reason == reason_oos_loh) ||
14947 (local_settings->reason == reason_alloc_loh))
14948 evaluate_elevation = FALSE;
14950 local_condemn_reasons->set_condition (gen_before_oom);
14955 if (is_induced_blocking (settings.reason) &&
14956 n_initial == max_generation
14957 IN_STRESS_HEAP( && !settings.stress_induced ))
14959 if (heap_number == 0)
14961 dprintf (GTC_LOG, ("induced - BLOCK"));
14964 *blocking_collection_p = TRUE;
14965 local_condemn_reasons->set_condition (gen_induced_fullgc_p);
14966 evaluate_elevation = FALSE;
14969 if (settings.reason == reason_induced_noforce)
14971 local_condemn_reasons->set_condition (gen_induced_noforce_p);
14972 evaluate_elevation = FALSE;
14976 if (evaluate_elevation && (low_ephemeral_space || high_memory_load || v_high_memory_load))
14978 *elevation_requested_p = TRUE;
14980 // if we are in high memory load and have consumed 10% of the gen2 budget, do a gen2 now.
14981 if (high_memory_load || v_high_memory_load)
14983 dynamic_data* dd_max = dynamic_data_of (max_generation);
14984 if (((float)dd_new_allocation (dd_max) / (float)dd_desired_allocation (dd_max)) < 0.9)
14986 dprintf (GTC_LOG, ("%Id left in gen2 alloc (%Id)",
14987 dd_new_allocation (dd_max), dd_desired_allocation (dd_max)));
14988 n = max_generation;
14989 local_condemn_reasons->set_condition (gen_almost_max_alloc);
14993 if (n <= max_generation)
14996 if (high_fragmentation)
14998 //elevate to max_generation
14999 n = max_generation;
15000 dprintf (GTC_LOG, ("h%d: f full", heap_number));
15002 #ifdef BACKGROUND_GC
15003 if (high_memory_load || v_high_memory_load)
15005 // For background GC we want to do blocking collections more eagerly because we don't
15006 // want to get into the situation where the memory load becomes high while we are in
15007 // a background GC and we'd have to wait for the background GC to finish to start
15008 // a blocking collection (right now the implemenation doesn't handle converting
15009 // a background GC to a blocking collection midway.
15010 dprintf (GTC_LOG, ("h%d: bgc - BLOCK", heap_number));
15011 *blocking_collection_p = TRUE;
15014 if (v_high_memory_load)
15016 dprintf (GTC_LOG, ("h%d: - BLOCK", heap_number));
15017 *blocking_collection_p = TRUE;
15019 #endif //BACKGROUND_GC
15023 n = max (n, max_generation - 1);
15024 dprintf (GTC_LOG, ("h%d: nf c %d", heap_number, n));
15031 if ((n == (max_generation - 1)) && (n_alloc < (max_generation -1)))
15033 dprintf (GTC_LOG, ("h%d: budget %d, check 2",
15034 heap_number, n_alloc));
15035 if (get_new_allocation (max_generation) <= 0)
15037 dprintf (GTC_LOG, ("h%d: budget alloc", heap_number));
15038 n = max_generation;
15039 local_condemn_reasons->set_condition (gen_max_gen1);
15043 //figure out if max_generation is too fragmented -> blocking collection
15044 if (n == max_generation)
15046 if (dt_high_frag_p (tuning_deciding_condemned_gen, n))
15048 dprintf (GTC_LOG, ("h%d: g%d too frag", heap_number, n));
15049 local_condemn_reasons->set_condition (gen_max_high_frag_p);
15050 if (local_settings->pause_mode != pause_sustained_low_latency)
15052 *blocking_collection_p = TRUE;
15057 #ifdef BACKGROUND_GC
15058 if (n == max_generation)
15060 if (heap_number == 0)
15062 BOOL bgc_heap_too_small = TRUE;
15063 size_t gen2size = 0;
15064 size_t gen3size = 0;
15065 #ifdef MULTIPLE_HEAPS
15066 for (int i = 0; i < n_heaps; i++)
15068 if (((g_heaps[i]->current_generation_size (max_generation)) > bgc_min_per_heap) ||
15069 ((g_heaps[i]->current_generation_size (max_generation + 1)) > bgc_min_per_heap))
15071 bgc_heap_too_small = FALSE;
15075 #else //MULTIPLE_HEAPS
15076 if ((current_generation_size (max_generation) > bgc_min_per_heap) ||
15077 (current_generation_size (max_generation + 1) > bgc_min_per_heap))
15079 bgc_heap_too_small = FALSE;
15081 #endif //MULTIPLE_HEAPS
15083 if (bgc_heap_too_small)
15085 dprintf (GTC_LOG, ("gen2 and gen3 too small"));
15088 // do not turn stress-induced collections into blocking GCs
15089 if (!settings.stress_induced)
15090 #endif //STRESS_HEAP
15092 *blocking_collection_p = TRUE;
15095 local_condemn_reasons->set_condition (gen_gen2_too_small);
15099 #endif //BACKGROUND_GC
15105 #ifdef BACKGROUND_GC
15106 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
15107 // generations to be collected,
15109 if (orig_gen != max_generation &&
15110 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
15112 *elevation_requested_p = FALSE;
15114 #endif //BACKGROUND_GC
15115 #endif //STRESS_HEAP
15119 fgm_result.available_pagefile_mb = (size_t)(available_page_file / (1024 * 1024));
15122 local_condemn_reasons->set_gen (gen_final_per_heap, n);
15123 get_gc_data_per_heap()->gen_to_condemn_reasons.init (local_condemn_reasons);
15126 local_condemn_reasons->print (heap_number);
15129 if ((local_settings->reason == reason_oos_soh) ||
15130 (local_settings->reason == reason_oos_loh))
15136 if (n == max_generation && GCToEEInterface::ForceFullGCToBeBlocking())
15138 #ifdef BACKGROUND_GC
15139 // do not turn stress-induced collections into blocking GCs, unless there
15140 // have already been more full BGCs than full NGCs
15142 // This exposes DevDiv 94129, so we'll leave this out for now
15143 if (!settings.stress_induced ||
15144 full_gc_counts[gc_type_blocking] <= full_gc_counts[gc_type_background])
15146 #endif // BACKGROUND_GC
15148 *blocking_collection_p = TRUE;
15156 #pragma warning(pop)
15160 size_t gc_heap::min_reclaim_fragmentation_threshold (uint32_t num_heaps)
15162 // if the memory load is higher, the threshold we'd want to collect gets lower.
15163 size_t min_mem_based_on_available =
15164 (500 - (settings.entry_memory_load - high_memory_load_th) * 40) * 1024 * 1024 / num_heaps;
15165 size_t ten_percent_size = (size_t)((float)generation_size (max_generation) * 0.10);
15166 uint64_t three_percent_mem = mem_one_percent * 3 / num_heaps;
15168 #ifdef SIMPLE_DPRINTF
15169 dprintf (GTC_LOG, ("min av: %Id, 10%% gen2: %Id, 3%% mem: %I64d",
15170 min_mem_based_on_available, ten_percent_size, three_percent_mem));
15171 #endif //SIMPLE_DPRINTF
15172 return (size_t)(min (min_mem_based_on_available, min (ten_percent_size, three_percent_mem)));
15176 uint64_t gc_heap::min_high_fragmentation_threshold(uint64_t available_mem, uint32_t num_heaps)
15178 return min (available_mem, (256*1024*1024)) / num_heaps;
15182 CORINFO_EXCEPTION_GC = 0xE0004743 // 'GC'
15186 #ifdef BACKGROUND_GC
15187 void gc_heap::init_background_gc ()
15189 //reset the allocation so foreground gc can allocate into older (max_generation) generation
15190 generation* gen = generation_of (max_generation);
15191 generation_allocation_pointer (gen)= 0;
15192 generation_allocation_limit (gen) = 0;
15193 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
15195 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
15197 //reset the plan allocation for each segment
15198 for (heap_segment* seg = generation_allocation_segment (gen); seg != ephemeral_heap_segment;
15199 seg = heap_segment_next_rw (seg))
15201 heap_segment_plan_allocated (seg) = heap_segment_allocated (seg);
15204 if (heap_number == 0)
15206 dprintf (2, ("heap%d: bgc lowest: %Ix, highest: %Ix",
15208 background_saved_lowest_address,
15209 background_saved_highest_address));
15212 gc_lh_block_event.Reset();
15215 #endif //BACKGROUND_GC
15218 void fire_drain_mark_list_event (size_t mark_list_objects)
15220 FIRE_EVENT(BGCDrainMark, mark_list_objects);
15224 void fire_revisit_event (size_t dirtied_pages,
15225 size_t marked_objects,
15226 BOOL large_objects_p)
15228 FIRE_EVENT(BGCRevisit, dirtied_pages, marked_objects, large_objects_p);
15232 void fire_overflow_event (uint8_t* overflow_min,
15233 uint8_t* overflow_max,
15234 size_t marked_objects,
15235 int large_objects_p)
15237 FIRE_EVENT(BGCOverflow, (uint64_t)overflow_min, (uint64_t)overflow_max, marked_objects, large_objects_p);
15240 void gc_heap::concurrent_print_time_delta (const char* msg)
15243 size_t current_time = GetHighPrecisionTimeStamp();
15244 size_t elapsed_time = current_time - time_bgc_last;
15245 time_bgc_last = current_time;
15247 dprintf (2, ("h%d: %s T %Id ms", heap_number, msg, elapsed_time));
15249 UNREFERENCED_PARAMETER(msg);
15253 void gc_heap::free_list_info (int gen_num, const char* msg)
15255 UNREFERENCED_PARAMETER(gen_num);
15256 #if defined (BACKGROUND_GC) && defined (TRACE_GC)
15257 dprintf (3, ("h%d: %s", heap_number, msg));
15258 for (int i = 0; i <= (max_generation + 1); i++)
15260 generation* gen = generation_of (i);
15261 if ((generation_allocation_size (gen) == 0) &&
15262 (generation_free_list_space (gen) == 0) &&
15263 (generation_free_obj_space (gen) == 0))
15265 // don't print if everything is 0.
15269 dprintf (3, ("h%d: g%d: a-%Id, fl-%Id, fo-%Id",
15271 generation_allocation_size (gen),
15272 generation_free_list_space (gen),
15273 generation_free_obj_space (gen)));
15277 UNREFERENCED_PARAMETER(msg);
15278 #endif // BACKGROUND_GC && TRACE_GC
15281 void gc_heap::update_collection_counts_for_no_gc()
15283 assert (settings.pause_mode == pause_no_gc);
15285 settings.condemned_generation = max_generation;
15286 #ifdef MULTIPLE_HEAPS
15287 for (int i = 0; i < n_heaps; i++)
15288 g_heaps[i]->update_collection_counts();
15289 #else //MULTIPLE_HEAPS
15290 update_collection_counts();
15291 #endif //MULTIPLE_HEAPS
15293 full_gc_counts[gc_type_blocking]++;
15296 BOOL gc_heap::should_proceed_with_gc()
15298 if (gc_heap::settings.pause_mode == pause_no_gc)
15300 if (current_no_gc_region_info.started)
15302 // The no_gc mode was already in progress yet we triggered another GC,
15303 // this effectively exits the no_gc mode.
15304 restore_data_for_no_gc();
15307 return should_proceed_for_no_gc();
15313 //internal part of gc used by the serial and concurrent version
15314 void gc_heap::gc1()
15316 #ifdef BACKGROUND_GC
15317 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15318 #endif //BACKGROUND_GC
15321 mark_time = plan_time = reloc_time = compact_time = sweep_time = 0;
15324 verify_soh_segment_list();
15326 int n = settings.condemned_generation;
15328 update_collection_counts ();
15330 #ifdef BACKGROUND_GC
15331 bgc_alloc_lock->check();
15332 #endif //BACKGROUND_GC
15334 free_list_info (max_generation, "beginning");
15336 vm_heap->GcCondemnedGeneration = settings.condemned_generation;
15338 assert (g_gc_card_table == card_table);
15340 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
15341 assert (g_gc_card_bundle_table == card_bundle_table);
15345 if (n == max_generation)
15347 gc_low = lowest_address;
15348 gc_high = highest_address;
15352 gc_low = generation_allocation_start (generation_of (n));
15353 gc_high = heap_segment_reserved (ephemeral_heap_segment);
15355 #ifdef BACKGROUND_GC
15356 if (settings.concurrent)
15359 time_bgc_last = GetHighPrecisionTimeStamp();
15362 FIRE_EVENT(BGCBegin);
15364 concurrent_print_time_delta ("BGC");
15366 //#ifdef WRITE_WATCH
15367 //reset_write_watch (FALSE);
15368 //#endif //WRITE_WATCH
15370 concurrent_print_time_delta ("RW");
15371 background_mark_phase();
15372 free_list_info (max_generation, "after mark phase");
15374 background_sweep();
15375 free_list_info (max_generation, "after sweep phase");
15378 #endif //BACKGROUND_GC
15380 mark_phase (n, FALSE);
15382 GCScan::GcRuntimeStructuresValid (FALSE);
15384 GCScan::GcRuntimeStructuresValid (TRUE);
15388 size_t end_gc_time = GetHighPrecisionTimeStamp();
15389 // printf ("generation: %d, elapsed time: %Id\n", n, end_gc_time - dd_time_clock (dynamic_data_of (0)));
15391 //adjust the allocation size from the pinned quantities.
15392 for (int gen_number = 0; gen_number <= min (max_generation,n+1); gen_number++)
15394 generation* gn = generation_of (gen_number);
15395 if (settings.compaction)
15397 generation_pinned_allocated (gn) += generation_pinned_allocation_compact_size (gn);
15398 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_compact_size (gn);
15402 generation_pinned_allocated (gn) += generation_pinned_allocation_sweep_size (gn);
15403 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_sweep_size (gn);
15405 generation_pinned_allocation_sweep_size (gn) = 0;
15406 generation_pinned_allocation_compact_size (gn) = 0;
15409 #ifdef BACKGROUND_GC
15410 if (settings.concurrent)
15412 dynamic_data* dd = dynamic_data_of (n);
15413 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15415 free_list_info (max_generation, "after computing new dynamic data");
15417 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
15419 for (int gen_number = 0; gen_number < max_generation; gen_number++)
15421 dprintf (2, ("end of BGC: gen%d new_alloc: %Id",
15422 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15423 current_gc_data_per_heap->gen_data[gen_number].size_after = generation_size (gen_number);
15424 current_gc_data_per_heap->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15425 current_gc_data_per_heap->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15429 #endif //BACKGROUND_GC
15431 free_list_info (max_generation, "end");
15432 for (int gen_number = 0; gen_number <= n; gen_number++)
15434 dynamic_data* dd = dynamic_data_of (gen_number);
15435 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15436 compute_new_dynamic_data (gen_number);
15439 if (n != max_generation)
15441 int gen_num_for_data = ((n < (max_generation - 1)) ? (n + 1) : (max_generation + 1));
15442 for (int gen_number = (n + 1); gen_number <= gen_num_for_data; gen_number++)
15444 get_gc_data_per_heap()->gen_data[gen_number].size_after = generation_size (gen_number);
15445 get_gc_data_per_heap()->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15446 get_gc_data_per_heap()->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15450 get_gc_data_per_heap()->maxgen_size_info.running_free_list_efficiency = (uint32_t)(generation_allocator_efficiency (generation_of (max_generation)) * 100);
15452 free_list_info (max_generation, "after computing new dynamic data");
15454 if (heap_number == 0)
15456 dprintf (GTC_LOG, ("GC#%d(gen%d) took %Idms",
15457 dd_collection_count (dynamic_data_of (0)),
15458 settings.condemned_generation,
15459 dd_gc_elapsed_time (dynamic_data_of (0))));
15462 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
15464 dprintf (2, ("end of FGC/NGC: gen%d new_alloc: %Id",
15465 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15469 if (n < max_generation)
15471 compute_promoted_allocation (1 + n);
15473 dynamic_data* dd = dynamic_data_of (1 + n);
15474 size_t new_fragmentation = generation_free_list_space (generation_of (1 + n)) +
15475 generation_free_obj_space (generation_of (1 + n));
15477 #ifdef BACKGROUND_GC
15478 if (current_c_gc_state != c_gc_state_planning)
15479 #endif //BACKGROUND_GC
15481 if (settings.promotion)
15483 dd_fragmentation (dd) = new_fragmentation;
15487 //assert (dd_fragmentation (dd) == new_fragmentation);
15492 #ifdef BACKGROUND_GC
15493 if (!settings.concurrent)
15494 #endif //BACKGROUND_GC
15496 #ifndef FEATURE_REDHAWK
15497 // GCToEEInterface::IsGCThread() always returns false on CoreRT, but this assert is useful in CoreCLR.
15498 assert(GCToEEInterface::IsGCThread());
15499 #endif // FEATURE_REDHAWK
15500 adjust_ephemeral_limits();
15503 #ifdef BACKGROUND_GC
15504 assert (ephemeral_low == generation_allocation_start (generation_of ( max_generation -1)));
15505 assert (ephemeral_high == heap_segment_reserved (ephemeral_heap_segment));
15506 #endif //BACKGROUND_GC
15508 if (fgn_maxgen_percent)
15510 if (settings.condemned_generation == (max_generation - 1))
15512 check_for_full_gc (max_generation - 1, 0);
15514 else if (settings.condemned_generation == max_generation)
15516 if (full_gc_approach_event_set
15517 #ifdef MULTIPLE_HEAPS
15518 && (heap_number == 0)
15519 #endif //MULTIPLE_HEAPS
15522 dprintf (2, ("FGN-GC: setting gen2 end event"));
15524 full_gc_approach_event.Reset();
15525 #ifdef BACKGROUND_GC
15526 // By definition WaitForFullGCComplete only succeeds if it's full, *blocking* GC, otherwise need to return N/A
15527 fgn_last_gc_was_concurrent = settings.concurrent ? TRUE : FALSE;
15528 #endif //BACKGROUND_GC
15529 full_gc_end_event.Set();
15530 full_gc_approach_event_set = false;
15535 #ifdef BACKGROUND_GC
15536 if (!settings.concurrent)
15537 #endif //BACKGROUND_GC
15539 //decide on the next allocation quantum
15540 if (alloc_contexts_used >= 1)
15542 allocation_quantum = Align (min ((size_t)CLR_SIZE,
15543 (size_t)max (1024, get_new_allocation (0) / (2 * alloc_contexts_used))),
15544 get_alignment_constant(FALSE));
15545 dprintf (3, ("New allocation quantum: %d(0x%Ix)", allocation_quantum, allocation_quantum));
15549 descr_generations (FALSE);
15551 verify_soh_segment_list();
15553 #ifdef BACKGROUND_GC
15554 add_to_history_per_heap();
15555 if (heap_number == 0)
15559 #endif // BACKGROUND_GC
15562 if (GCStatistics::Enabled() && heap_number == 0)
15563 g_GCStatistics.AddGCStats(settings,
15564 dd_gc_elapsed_time(dynamic_data_of(settings.condemned_generation)));
15568 fprintf (stdout, "%d,%d,%d,%d,%d,%d\n",
15569 n, mark_time, plan_time, reloc_time, compact_time, sweep_time);
15572 #ifdef BACKGROUND_GC
15573 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15574 #endif //BACKGROUND_GC
15576 #if defined(VERIFY_HEAP) || (defined (FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
15579 // Note that right now g_pConfig->GetHeapVerifyLevel always returns the same
15580 // value. If we ever allow randomly adjusting this as the process runs,
15581 // we cannot call it this way as joins need to match - we must have the same
15582 // value for all heaps like we do with bgc_heap_walk_for_etw_p.
15583 || (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
15585 #if defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC)
15586 || (bgc_heap_walk_for_etw_p && settings.concurrent)
15590 #ifdef BACKGROUND_GC
15591 bool cooperative_mode = true;
15593 if (settings.concurrent)
15595 cooperative_mode = enable_preemptive ();
15597 #ifdef MULTIPLE_HEAPS
15598 bgc_t_join.join(this, gc_join_suspend_ee_verify);
15599 if (bgc_t_join.joined())
15601 bgc_threads_sync_event.Reset();
15603 dprintf(2, ("Joining BGC threads to suspend EE for verify heap"));
15604 bgc_t_join.restart();
15606 if (heap_number == 0)
15609 bgc_threads_sync_event.Set();
15613 bgc_threads_sync_event.Wait(INFINITE, FALSE);
15614 dprintf (2, ("bgc_threads_sync_event is signalled"));
15618 #endif //MULTIPLE_HEAPS
15620 //fix the allocation area so verify_heap can proceed.
15621 fix_allocation_contexts (FALSE);
15623 #endif //BACKGROUND_GC
15625 #ifdef BACKGROUND_GC
15626 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15627 #ifdef FEATURE_EVENT_TRACE
15628 if (bgc_heap_walk_for_etw_p && settings.concurrent)
15630 GCToEEInterface::DiagWalkBGCSurvivors(__this);
15632 #ifdef MULTIPLE_HEAPS
15633 bgc_t_join.join(this, gc_join_after_profiler_heap_walk);
15634 if (bgc_t_join.joined())
15636 bgc_t_join.restart();
15638 #endif // MULTIPLE_HEAPS
15640 #endif // FEATURE_EVENT_TRACE
15641 #endif //BACKGROUND_GC
15644 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
15645 verify_heap (FALSE);
15646 #endif // VERIFY_HEAP
15648 #ifdef BACKGROUND_GC
15649 if (settings.concurrent)
15651 repair_allocation_contexts (TRUE);
15653 #ifdef MULTIPLE_HEAPS
15654 bgc_t_join.join(this, gc_join_restart_ee_verify);
15655 if (bgc_t_join.joined())
15657 bgc_threads_sync_event.Reset();
15659 dprintf(2, ("Joining BGC threads to restart EE after verify heap"));
15660 bgc_t_join.restart();
15662 if (heap_number == 0)
15665 bgc_threads_sync_event.Set();
15669 bgc_threads_sync_event.Wait(INFINITE, FALSE);
15670 dprintf (2, ("bgc_threads_sync_event is signalled"));
15674 #endif //MULTIPLE_HEAPS
15676 disable_preemptive (cooperative_mode);
15678 #endif //BACKGROUND_GC
15680 #endif // defined(VERIFY_HEAP) || (defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
15682 #ifdef MULTIPLE_HEAPS
15683 if (!settings.concurrent)
15685 gc_t_join.join(this, gc_join_done);
15686 if (gc_t_join.joined ())
15688 gc_heap::internal_gc_done = false;
15690 //equalize the new desired size of the generations
15691 int limit = settings.condemned_generation;
15692 if (limit == max_generation)
15694 limit = max_generation+1;
15696 for (int gen = 0; gen <= limit; gen++)
15698 size_t total_desired = 0;
15700 for (int i = 0; i < gc_heap::n_heaps; i++)
15702 gc_heap* hp = gc_heap::g_heaps[i];
15703 dynamic_data* dd = hp->dynamic_data_of (gen);
15704 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
15705 if (temp_total_desired < total_desired)
15708 total_desired = (size_t)MAX_PTR;
15711 total_desired = temp_total_desired;
15714 size_t desired_per_heap = Align (total_desired/gc_heap::n_heaps,
15715 get_alignment_constant ((gen != (max_generation+1))));
15719 #if 1 //subsumed by the linear allocation model
15720 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
15721 // apply some smoothing.
15722 static size_t smoothed_desired_per_heap = 0;
15723 size_t smoothing = 3; // exponential smoothing factor
15724 if (smoothing > VolatileLoad(&settings.gc_index))
15725 smoothing = VolatileLoad(&settings.gc_index);
15726 smoothed_desired_per_heap = desired_per_heap / smoothing + ((smoothed_desired_per_heap / smoothing) * (smoothing-1));
15727 dprintf (1, ("sn = %Id n = %Id", smoothed_desired_per_heap, desired_per_heap));
15728 desired_per_heap = Align(smoothed_desired_per_heap, get_alignment_constant (true));
15731 // if desired_per_heap is close to min_gc_size, trim it
15732 // down to min_gc_size to stay in the cache
15733 gc_heap* hp = gc_heap::g_heaps[0];
15734 dynamic_data* dd = hp->dynamic_data_of (gen);
15735 size_t min_gc_size = dd_min_size(dd);
15736 // if min GC size larger than true on die cache, then don't bother
15737 // limiting the desired size
15738 if ((min_gc_size <= GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)) &&
15739 desired_per_heap <= 2*min_gc_size)
15741 desired_per_heap = min_gc_size;
15744 desired_per_heap = joined_youngest_desired (desired_per_heap);
15745 dprintf (2, ("final gen0 new_alloc: %Id", desired_per_heap));
15748 gc_data_global.final_youngest_desired = desired_per_heap;
15750 #if 1 //subsumed by the linear allocation model
15751 if (gen == (max_generation + 1))
15753 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
15754 // apply some smoothing.
15755 static size_t smoothed_desired_per_heap_loh = 0;
15756 size_t smoothing = 3; // exponential smoothing factor
15757 size_t loh_count = dd_collection_count (dynamic_data_of (max_generation));
15758 if (smoothing > loh_count)
15759 smoothing = loh_count;
15760 smoothed_desired_per_heap_loh = desired_per_heap / smoothing + ((smoothed_desired_per_heap_loh / smoothing) * (smoothing-1));
15761 dprintf( 2, ("smoothed_desired_per_heap_loh = %Id desired_per_heap = %Id", smoothed_desired_per_heap_loh, desired_per_heap));
15762 desired_per_heap = Align(smoothed_desired_per_heap_loh, get_alignment_constant (false));
15765 for (int i = 0; i < gc_heap::n_heaps; i++)
15767 gc_heap* hp = gc_heap::g_heaps[i];
15768 dynamic_data* dd = hp->dynamic_data_of (gen);
15769 dd_desired_allocation (dd) = desired_per_heap;
15770 dd_gc_new_allocation (dd) = desired_per_heap;
15771 dd_new_allocation (dd) = desired_per_heap;
15775 hp->fgn_last_alloc = desired_per_heap;
15780 #ifdef FEATURE_LOH_COMPACTION
15781 BOOL all_heaps_compacted_p = TRUE;
15782 #endif //FEATURE_LOH_COMPACTION
15783 for (int i = 0; i < gc_heap::n_heaps; i++)
15785 gc_heap* hp = gc_heap::g_heaps[i];
15786 hp->decommit_ephemeral_segment_pages();
15787 hp->rearrange_large_heap_segments();
15788 #ifdef FEATURE_LOH_COMPACTION
15789 all_heaps_compacted_p &= hp->loh_compacted_p;
15790 #endif //FEATURE_LOH_COMPACTION
15793 #ifdef FEATURE_LOH_COMPACTION
15794 check_loh_compact_mode (all_heaps_compacted_p);
15795 #endif //FEATURE_LOH_COMPACTION
15799 gc_t_join.restart();
15801 alloc_context_count = 0;
15802 heap_select::mark_heap (heap_number);
15806 gc_data_global.final_youngest_desired =
15807 dd_desired_allocation (dynamic_data_of (0));
15809 check_loh_compact_mode (loh_compacted_p);
15811 decommit_ephemeral_segment_pages();
15814 if (!(settings.concurrent))
15816 rearrange_large_heap_segments();
15820 #ifdef BACKGROUND_GC
15821 recover_bgc_settings();
15822 #endif //BACKGROUND_GC
15823 #endif //MULTIPLE_HEAPS
15826 void gc_heap::save_data_for_no_gc()
15828 current_no_gc_region_info.saved_pause_mode = settings.pause_mode;
15829 #ifdef MULTIPLE_HEAPS
15830 // This is to affect heap balancing.
15831 for (int i = 0; i < n_heaps; i++)
15833 current_no_gc_region_info.saved_gen0_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (0));
15834 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = min_balance_threshold;
15835 current_no_gc_region_info.saved_gen3_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1));
15836 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = 0;
15838 #endif //MULTIPLE_HEAPS
15841 void gc_heap::restore_data_for_no_gc()
15843 gc_heap::settings.pause_mode = current_no_gc_region_info.saved_pause_mode;
15844 #ifdef MULTIPLE_HEAPS
15845 for (int i = 0; i < n_heaps; i++)
15847 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = current_no_gc_region_info.saved_gen0_min_size;
15848 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = current_no_gc_region_info.saved_gen3_min_size;
15850 #endif //MULTIPLE_HEAPS
15853 start_no_gc_region_status gc_heap::prepare_for_no_gc_region (uint64_t total_size,
15854 BOOL loh_size_known,
15856 BOOL disallow_full_blocking)
15858 if (current_no_gc_region_info.started)
15860 return start_no_gc_in_progress;
15863 start_no_gc_region_status status = start_no_gc_success;
15865 save_data_for_no_gc();
15866 settings.pause_mode = pause_no_gc;
15867 current_no_gc_region_info.start_status = start_no_gc_success;
15869 uint64_t allocation_no_gc_loh = 0;
15870 uint64_t allocation_no_gc_soh = 0;
15871 assert(total_size != 0);
15872 if (loh_size_known)
15874 assert(loh_size != 0);
15875 assert(loh_size <= total_size);
15876 allocation_no_gc_loh = loh_size;
15877 allocation_no_gc_soh = total_size - loh_size;
15881 allocation_no_gc_soh = total_size;
15882 allocation_no_gc_loh = total_size;
15885 int soh_align_const = get_alignment_constant (TRUE);
15886 size_t max_soh_allocated = soh_segment_size - segment_info_size - eph_gen_starts_size;
15887 size_t size_per_heap = 0;
15888 const double scale_factor = 1.05;
15891 #ifdef MULTIPLE_HEAPS
15892 num_heaps = n_heaps;
15893 #endif // MULTIPLE_HEAPS
15895 uint64_t total_allowed_soh_allocation = max_soh_allocated * num_heaps;
15897 // In theory, the upper limit here is the physical memory of the machine, not
15898 // SIZE_T_MAX. This is not true today because total_physical_mem can be
15899 // larger than SIZE_T_MAX if running in wow64 on a machine with more than
15900 // 4GB of RAM. Once Local GC code divergence is resolved and code is flowing
15901 // more freely between branches, it would be good to clean this up to use
15902 // total_physical_mem instead of SIZE_T_MAX.
15903 assert(total_allowed_soh_allocation <= SIZE_T_MAX);
15904 uint64_t total_allowed_loh_allocation = SIZE_T_MAX;
15905 uint64_t total_allowed_soh_alloc_scaled = allocation_no_gc_soh > 0 ? static_cast<uint64_t>(total_allowed_soh_allocation / scale_factor) : 0;
15906 uint64_t total_allowed_loh_alloc_scaled = allocation_no_gc_loh > 0 ? static_cast<uint64_t>(total_allowed_loh_allocation / scale_factor) : 0;
15908 if (allocation_no_gc_soh > total_allowed_soh_alloc_scaled ||
15909 allocation_no_gc_loh > total_allowed_loh_alloc_scaled)
15911 status = start_no_gc_too_large;
15915 if (allocation_no_gc_soh > 0)
15917 allocation_no_gc_soh = static_cast<uint64_t>(allocation_no_gc_soh * scale_factor);
15918 allocation_no_gc_soh = min (allocation_no_gc_soh, total_allowed_soh_alloc_scaled);
15921 if (allocation_no_gc_loh > 0)
15923 allocation_no_gc_loh = static_cast<uint64_t>(allocation_no_gc_loh * scale_factor);
15924 allocation_no_gc_loh = min (allocation_no_gc_loh, total_allowed_loh_alloc_scaled);
15927 if (disallow_full_blocking)
15928 current_no_gc_region_info.minimal_gc_p = TRUE;
15930 if (allocation_no_gc_soh != 0)
15932 current_no_gc_region_info.soh_allocation_size = static_cast<size_t>(allocation_no_gc_soh);
15933 size_per_heap = current_no_gc_region_info.soh_allocation_size;
15934 #ifdef MULTIPLE_HEAPS
15935 size_per_heap /= n_heaps;
15936 for (int i = 0; i < n_heaps; i++)
15938 // due to heap balancing we need to allow some room before we even look to balance to another heap.
15939 g_heaps[i]->soh_allocation_no_gc = min (Align ((size_per_heap + min_balance_threshold), soh_align_const), max_soh_allocated);
15941 #else //MULTIPLE_HEAPS
15942 soh_allocation_no_gc = min (Align (size_per_heap, soh_align_const), max_soh_allocated);
15943 #endif //MULTIPLE_HEAPS
15946 if (allocation_no_gc_loh != 0)
15948 current_no_gc_region_info.loh_allocation_size = static_cast<size_t>(allocation_no_gc_loh);
15949 size_per_heap = current_no_gc_region_info.loh_allocation_size;
15950 #ifdef MULTIPLE_HEAPS
15951 size_per_heap /= n_heaps;
15952 for (int i = 0; i < n_heaps; i++)
15953 g_heaps[i]->loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
15954 #else //MULTIPLE_HEAPS
15955 loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
15956 #endif //MULTIPLE_HEAPS
15960 if (status != start_no_gc_success)
15961 restore_data_for_no_gc();
15965 void gc_heap::handle_failure_for_no_gc()
15967 gc_heap::restore_data_for_no_gc();
15968 // sets current_no_gc_region_info.started to FALSE here.
15969 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
15972 start_no_gc_region_status gc_heap::get_start_no_gc_region_status()
15974 return current_no_gc_region_info.start_status;
15977 void gc_heap::record_gcs_during_no_gc()
15979 if (current_no_gc_region_info.started)
15981 current_no_gc_region_info.num_gcs++;
15982 if (is_induced (settings.reason))
15983 current_no_gc_region_info.num_gcs_induced++;
15987 BOOL gc_heap::find_loh_free_for_no_gc()
15989 allocator* loh_allocator = generation_allocator (generation_of (max_generation + 1));
15990 size_t sz_list = loh_allocator->first_bucket_size();
15991 size_t size = loh_allocation_no_gc;
15992 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
15994 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
15996 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
15999 size_t free_list_size = unused_array_size(free_list);
16001 if (free_list_size > loh_allocation_no_gc)
16003 dprintf (3, ("free item %Ix(%Id) for no gc", (size_t)free_list, free_list_size));
16007 free_list = free_list_slot (free_list);
16010 sz_list = sz_list * 2;
16016 BOOL gc_heap::find_loh_space_for_no_gc()
16018 saved_loh_segment_no_gc = 0;
16020 if (find_loh_free_for_no_gc())
16023 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16027 size_t remaining = heap_segment_reserved (seg) - heap_segment_allocated (seg);
16028 if (remaining >= loh_allocation_no_gc)
16030 saved_loh_segment_no_gc = seg;
16033 seg = heap_segment_next (seg);
16036 if (!saved_loh_segment_no_gc && current_no_gc_region_info.minimal_gc_p)
16038 // If no full GC is allowed, we try to get a new seg right away.
16039 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc)
16040 #ifdef MULTIPLE_HEAPS
16042 #endif //MULTIPLE_HEAPS
16046 return (saved_loh_segment_no_gc != 0);
16049 BOOL gc_heap::loh_allocated_for_no_gc()
16051 if (!saved_loh_segment_no_gc)
16054 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16057 if (seg == saved_loh_segment_no_gc)
16061 seg = heap_segment_next (seg);
16067 BOOL gc_heap::commit_loh_for_no_gc (heap_segment* seg)
16069 uint8_t* end_committed = heap_segment_allocated (seg) + loh_allocation_no_gc;
16070 assert (end_committed <= heap_segment_reserved (seg));
16071 return (grow_heap_segment (seg, end_committed));
16074 void gc_heap::thread_no_gc_loh_segments()
16076 #ifdef MULTIPLE_HEAPS
16077 for (int i = 0; i < n_heaps; i++)
16079 gc_heap* hp = g_heaps[i];
16080 if (hp->loh_allocated_for_no_gc())
16082 hp->thread_loh_segment (hp->saved_loh_segment_no_gc);
16083 hp->saved_loh_segment_no_gc = 0;
16086 #else //MULTIPLE_HEAPS
16087 if (loh_allocated_for_no_gc())
16089 thread_loh_segment (saved_loh_segment_no_gc);
16090 saved_loh_segment_no_gc = 0;
16092 #endif //MULTIPLE_HEAPS
16095 void gc_heap::set_loh_allocations_for_no_gc()
16097 if (current_no_gc_region_info.loh_allocation_size != 0)
16099 dynamic_data* dd = dynamic_data_of (max_generation + 1);
16100 dd_new_allocation (dd) = loh_allocation_no_gc;
16101 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16105 void gc_heap::set_soh_allocations_for_no_gc()
16107 if (current_no_gc_region_info.soh_allocation_size != 0)
16109 dynamic_data* dd = dynamic_data_of (0);
16110 dd_new_allocation (dd) = soh_allocation_no_gc;
16111 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16112 #ifdef MULTIPLE_HEAPS
16113 alloc_context_count = 0;
16114 #endif //MULTIPLE_HEAPS
16118 void gc_heap::set_allocations_for_no_gc()
16120 #ifdef MULTIPLE_HEAPS
16121 for (int i = 0; i < n_heaps; i++)
16123 gc_heap* hp = g_heaps[i];
16124 hp->set_loh_allocations_for_no_gc();
16125 hp->set_soh_allocations_for_no_gc();
16127 #else //MULTIPLE_HEAPS
16128 set_loh_allocations_for_no_gc();
16129 set_soh_allocations_for_no_gc();
16130 #endif //MULTIPLE_HEAPS
16133 BOOL gc_heap::should_proceed_for_no_gc()
16135 BOOL gc_requested = FALSE;
16136 BOOL loh_full_gc_requested = FALSE;
16137 BOOL soh_full_gc_requested = FALSE;
16138 BOOL no_gc_requested = FALSE;
16139 BOOL get_new_loh_segments = FALSE;
16141 if (current_no_gc_region_info.soh_allocation_size)
16143 #ifdef MULTIPLE_HEAPS
16144 for (int i = 0; i < n_heaps; i++)
16146 gc_heap* hp = g_heaps[i];
16147 if ((size_t)(heap_segment_reserved (hp->ephemeral_heap_segment) - hp->alloc_allocated) < hp->soh_allocation_no_gc)
16149 gc_requested = TRUE;
16153 #else //MULTIPLE_HEAPS
16154 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated) < soh_allocation_no_gc)
16155 gc_requested = TRUE;
16156 #endif //MULTIPLE_HEAPS
16160 #ifdef MULTIPLE_HEAPS
16161 for (int i = 0; i < n_heaps; i++)
16163 gc_heap* hp = g_heaps[i];
16164 if (!(hp->grow_heap_segment (hp->ephemeral_heap_segment, (hp->alloc_allocated + hp->soh_allocation_no_gc))))
16166 soh_full_gc_requested = TRUE;
16170 #else //MULTIPLE_HEAPS
16171 if (!grow_heap_segment (ephemeral_heap_segment, (alloc_allocated + soh_allocation_no_gc)))
16172 soh_full_gc_requested = TRUE;
16173 #endif //MULTIPLE_HEAPS
16177 if (!current_no_gc_region_info.minimal_gc_p && gc_requested)
16179 soh_full_gc_requested = TRUE;
16182 no_gc_requested = !(soh_full_gc_requested || gc_requested);
16184 if (soh_full_gc_requested && current_no_gc_region_info.minimal_gc_p)
16186 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16190 if (!soh_full_gc_requested && current_no_gc_region_info.loh_allocation_size)
16192 // Check to see if we have enough reserved space.
16193 #ifdef MULTIPLE_HEAPS
16194 for (int i = 0; i < n_heaps; i++)
16196 gc_heap* hp = g_heaps[i];
16197 if (!hp->find_loh_space_for_no_gc())
16199 loh_full_gc_requested = TRUE;
16203 #else //MULTIPLE_HEAPS
16204 if (!find_loh_space_for_no_gc())
16205 loh_full_gc_requested = TRUE;
16206 #endif //MULTIPLE_HEAPS
16208 // Check to see if we have committed space.
16209 if (!loh_full_gc_requested)
16211 #ifdef MULTIPLE_HEAPS
16212 for (int i = 0; i < n_heaps; i++)
16214 gc_heap* hp = g_heaps[i];
16215 if (hp->saved_loh_segment_no_gc &&!hp->commit_loh_for_no_gc (hp->saved_loh_segment_no_gc))
16217 loh_full_gc_requested = TRUE;
16221 #else //MULTIPLE_HEAPS
16222 if (saved_loh_segment_no_gc && !commit_loh_for_no_gc (saved_loh_segment_no_gc))
16223 loh_full_gc_requested = TRUE;
16224 #endif //MULTIPLE_HEAPS
16228 if (loh_full_gc_requested || soh_full_gc_requested)
16230 if (current_no_gc_region_info.minimal_gc_p)
16231 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16234 no_gc_requested = !(loh_full_gc_requested || soh_full_gc_requested || gc_requested);
16236 if (current_no_gc_region_info.start_status == start_no_gc_success)
16238 if (no_gc_requested)
16239 set_allocations_for_no_gc();
16244 if ((current_no_gc_region_info.start_status == start_no_gc_success) && !no_gc_requested)
16248 // We are done with starting the no_gc_region.
16249 current_no_gc_region_info.started = TRUE;
16254 end_no_gc_region_status gc_heap::end_no_gc_region()
16256 dprintf (1, ("end no gc called"));
16258 end_no_gc_region_status status = end_no_gc_success;
16260 if (!(current_no_gc_region_info.started))
16261 status = end_no_gc_not_in_progress;
16262 if (current_no_gc_region_info.num_gcs_induced)
16263 status = end_no_gc_induced;
16264 else if (current_no_gc_region_info.num_gcs)
16265 status = end_no_gc_alloc_exceeded;
16267 if (settings.pause_mode == pause_no_gc)
16268 restore_data_for_no_gc();
16270 // sets current_no_gc_region_info.started to FALSE here.
16271 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
16277 void gc_heap::update_collection_counts ()
16279 dynamic_data* dd0 = dynamic_data_of (0);
16280 dd_gc_clock (dd0) += 1;
16282 size_t now = GetHighPrecisionTimeStamp();
16284 for (int i = 0; i <= settings.condemned_generation;i++)
16286 dynamic_data* dd = dynamic_data_of (i);
16287 dd_collection_count (dd)++;
16288 //this is needed by the linear allocation model
16289 if (i == max_generation)
16290 dd_collection_count (dynamic_data_of (max_generation+1))++;
16291 dd_gc_clock (dd) = dd_gc_clock (dd0);
16292 dd_time_clock (dd) = now;
16296 BOOL gc_heap::expand_soh_with_minimal_gc()
16298 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) >= soh_allocation_no_gc)
16301 heap_segment* new_seg = soh_get_segment_to_expand();
16304 if (g_gc_card_table != card_table)
16305 copy_brick_card_table();
16307 settings.promotion = TRUE;
16308 settings.demotion = FALSE;
16309 ephemeral_promotion = TRUE;
16310 int condemned_gen_number = max_generation - 1;
16312 generation* gen = 0;
16313 int align_const = get_alignment_constant (TRUE);
16315 for (int i = 0; i <= condemned_gen_number; i++)
16317 gen = generation_of (i);
16318 saved_ephemeral_plan_start[i] = generation_allocation_start (gen);
16319 saved_ephemeral_plan_start_size[i] = Align (size (generation_allocation_start (gen)), align_const);
16322 // We do need to clear the bricks here as we are converting a bunch of ephemeral objects to gen2
16323 // and need to make sure that there are no left over bricks from the previous GCs for the space
16324 // we just used for gen0 allocation. We will need to go through the bricks for these objects for
16325 // ephemeral GCs later.
16326 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
16327 b < brick_of (align_on_brick (heap_segment_allocated (ephemeral_heap_segment)));
16333 size_t ephemeral_size = (heap_segment_allocated (ephemeral_heap_segment) -
16334 generation_allocation_start (generation_of (max_generation - 1)));
16335 heap_segment_next (ephemeral_heap_segment) = new_seg;
16336 ephemeral_heap_segment = new_seg;
16337 uint8_t* start = heap_segment_mem (ephemeral_heap_segment);
16339 for (int i = condemned_gen_number; i >= 0; i--)
16341 gen = generation_of (i);
16342 size_t gen_start_size = Align (min_obj_size);
16343 make_generation (generation_table[i], ephemeral_heap_segment, start, 0);
16344 generation_plan_allocation_start (gen) = start;
16345 generation_plan_allocation_start_size (gen) = gen_start_size;
16346 start += gen_start_size;
16348 heap_segment_used (ephemeral_heap_segment) = start - plug_skew;
16349 heap_segment_plan_allocated (ephemeral_heap_segment) = start;
16351 fix_generation_bounds (condemned_gen_number, generation_of (0));
16353 dd_gc_new_allocation (dynamic_data_of (max_generation)) -= ephemeral_size;
16354 dd_new_allocation (dynamic_data_of (max_generation)) = dd_gc_new_allocation (dynamic_data_of (max_generation));
16356 adjust_ephemeral_limits();
16363 // Only to be done on the thread that calls restart in a join for server GC
16364 // and reset the oom status per heap.
16365 void gc_heap::check_and_set_no_gc_oom()
16367 #ifdef MULTIPLE_HEAPS
16368 for (int i = 0; i < n_heaps; i++)
16370 gc_heap* hp = g_heaps[i];
16371 if (hp->no_gc_oom_p)
16373 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16374 hp->no_gc_oom_p = false;
16380 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16381 no_gc_oom_p = false;
16383 #endif //MULTIPLE_HEAPS
16386 void gc_heap::allocate_for_no_gc_after_gc()
16388 if (current_no_gc_region_info.minimal_gc_p)
16389 repair_allocation_contexts (TRUE);
16391 no_gc_oom_p = false;
16393 if (current_no_gc_region_info.start_status != start_no_gc_no_memory)
16395 if (current_no_gc_region_info.soh_allocation_size != 0)
16397 if (((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) < soh_allocation_no_gc) ||
16398 (!grow_heap_segment (ephemeral_heap_segment, (heap_segment_allocated (ephemeral_heap_segment) + soh_allocation_no_gc))))
16400 no_gc_oom_p = true;
16403 #ifdef MULTIPLE_HEAPS
16404 gc_t_join.join(this, gc_join_after_commit_soh_no_gc);
16405 if (gc_t_join.joined())
16407 #endif //MULTIPLE_HEAPS
16409 check_and_set_no_gc_oom();
16411 #ifdef MULTIPLE_HEAPS
16412 gc_t_join.restart();
16414 #endif //MULTIPLE_HEAPS
16417 if ((current_no_gc_region_info.start_status == start_no_gc_success) &&
16418 !(current_no_gc_region_info.minimal_gc_p) &&
16419 (current_no_gc_region_info.loh_allocation_size != 0))
16421 gc_policy = policy_compact;
16422 saved_loh_segment_no_gc = 0;
16424 if (!find_loh_free_for_no_gc())
16426 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16427 BOOL found_seg_p = FALSE;
16430 if ((size_t)(heap_segment_reserved (seg) - heap_segment_allocated (seg)) >= loh_allocation_no_gc)
16432 found_seg_p = TRUE;
16433 if (!commit_loh_for_no_gc (seg))
16435 no_gc_oom_p = true;
16439 seg = heap_segment_next (seg);
16443 gc_policy = policy_expand;
16446 #ifdef MULTIPLE_HEAPS
16447 gc_t_join.join(this, gc_join_expand_loh_no_gc);
16448 if (gc_t_join.joined())
16450 check_and_set_no_gc_oom();
16452 if (current_no_gc_region_info.start_status == start_no_gc_success)
16454 for (int i = 0; i < n_heaps; i++)
16456 gc_heap* hp = g_heaps[i];
16457 if (hp->gc_policy == policy_expand)
16459 hp->saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc), hp);
16460 if (!(hp->saved_loh_segment_no_gc))
16462 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16469 gc_t_join.restart();
16471 #else //MULTIPLE_HEAPS
16472 check_and_set_no_gc_oom();
16474 if ((current_no_gc_region_info.start_status == start_no_gc_success) && (gc_policy == policy_expand))
16476 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc));
16477 if (!saved_loh_segment_no_gc)
16478 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16480 #endif //MULTIPLE_HEAPS
16482 if ((current_no_gc_region_info.start_status == start_no_gc_success) && saved_loh_segment_no_gc)
16484 if (!commit_loh_for_no_gc (saved_loh_segment_no_gc))
16486 no_gc_oom_p = true;
16492 #ifdef MULTIPLE_HEAPS
16493 gc_t_join.join(this, gc_join_final_no_gc);
16494 if (gc_t_join.joined())
16496 #endif //MULTIPLE_HEAPS
16498 check_and_set_no_gc_oom();
16500 if (current_no_gc_region_info.start_status == start_no_gc_success)
16502 set_allocations_for_no_gc();
16503 current_no_gc_region_info.started = TRUE;
16506 #ifdef MULTIPLE_HEAPS
16507 gc_t_join.restart();
16509 #endif //MULTIPLE_HEAPS
16512 void gc_heap::init_records()
16514 memset (&gc_data_per_heap, 0, sizeof (gc_data_per_heap));
16515 gc_data_per_heap.heap_index = heap_number;
16516 if (heap_number == 0)
16517 memset (&gc_data_global, 0, sizeof (gc_data_global));
16519 #ifdef GC_CONFIG_DRIVEN
16520 memset (interesting_data_per_gc, 0, sizeof (interesting_data_per_gc));
16521 #endif //GC_CONFIG_DRIVEN
16524 int gc_heap::garbage_collect (int n)
16526 //reset the number of alloc contexts
16527 alloc_contexts_used = 0;
16529 fix_allocation_contexts (TRUE);
16530 #ifdef MULTIPLE_HEAPS
16532 gc_t_join.start_ts(this);
16533 #endif //JOIN_STATS
16534 clear_gen0_bricks();
16535 #endif //MULTIPLE_HEAPS
16537 if ((settings.pause_mode == pause_no_gc) && current_no_gc_region_info.minimal_gc_p)
16539 #ifdef MULTIPLE_HEAPS
16540 gc_t_join.join(this, gc_join_minimal_gc);
16541 if (gc_t_join.joined())
16543 #endif //MULTIPLE_HEAPS
16545 #ifdef MULTIPLE_HEAPS
16546 // this is serialized because we need to get a segment
16547 for (int i = 0; i < n_heaps; i++)
16549 if (!(g_heaps[i]->expand_soh_with_minimal_gc()))
16550 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16553 if (!expand_soh_with_minimal_gc())
16554 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16555 #endif //MULTIPLE_HEAPS
16557 update_collection_counts_for_no_gc();
16559 #ifdef MULTIPLE_HEAPS
16560 gc_t_join.restart();
16562 #endif //MULTIPLE_HEAPS
16568 memset (&fgm_result, 0, sizeof (fgm_result));
16570 settings.reason = gc_trigger_reason;
16571 verify_pinned_queue_p = FALSE;
16573 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
16574 num_pinned_objects = 0;
16575 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
16578 if (settings.reason == reason_gcstress)
16580 settings.reason = reason_induced;
16581 settings.stress_induced = TRUE;
16583 #endif // STRESS_HEAP
16585 #ifdef MULTIPLE_HEAPS
16586 //align all heaps on the max generation to condemn
16587 dprintf (3, ("Joining for max generation to condemn"));
16588 condemned_generation_num = generation_to_condemn (n,
16589 &blocking_collection,
16590 &elevation_requested,
16592 gc_t_join.join(this, gc_join_generation_determined);
16593 if (gc_t_join.joined())
16594 #endif //MULTIPLE_HEAPS
16596 #ifdef MULTIPLE_HEAPS
16597 #if !defined(SEG_MAPPING_TABLE) && !defined(FEATURE_BASICFREEZE)
16598 //delete old slots from the segment table
16599 seg_table->delete_old_slots();
16600 #endif //!SEG_MAPPING_TABLE && !FEATURE_BASICFREEZE
16601 for (int i = 0; i < n_heaps; i++)
16603 //copy the card and brick tables
16604 if (g_gc_card_table != g_heaps[i]->card_table)
16606 g_heaps[i]->copy_brick_card_table();
16609 g_heaps[i]->rearrange_large_heap_segments();
16610 if (!recursive_gc_sync::background_running_p())
16612 g_heaps[i]->rearrange_small_heap_segments();
16615 #else //MULTIPLE_HEAPS
16616 #ifdef BACKGROUND_GC
16617 //delete old slots from the segment table
16618 #if !defined(SEG_MAPPING_TABLE) && !defined(FEATURE_BASICFREEZE)
16619 seg_table->delete_old_slots();
16620 #endif //!SEG_MAPPING_TABLE && !FEATURE_BASICFREEZE
16621 rearrange_large_heap_segments();
16622 if (!recursive_gc_sync::background_running_p())
16624 rearrange_small_heap_segments();
16626 #endif //BACKGROUND_GC
16627 // check for card table growth
16628 if (g_gc_card_table != card_table)
16629 copy_brick_card_table();
16631 #endif //MULTIPLE_HEAPS
16633 BOOL should_evaluate_elevation = FALSE;
16634 BOOL should_do_blocking_collection = FALSE;
16636 #ifdef MULTIPLE_HEAPS
16637 int gen_max = condemned_generation_num;
16638 for (int i = 0; i < n_heaps; i++)
16640 if (gen_max < g_heaps[i]->condemned_generation_num)
16641 gen_max = g_heaps[i]->condemned_generation_num;
16642 if ((!should_evaluate_elevation) && (g_heaps[i]->elevation_requested))
16643 should_evaluate_elevation = TRUE;
16644 if ((!should_do_blocking_collection) && (g_heaps[i]->blocking_collection))
16645 should_do_blocking_collection = TRUE;
16648 settings.condemned_generation = gen_max;
16649 #else //MULTIPLE_HEAPS
16650 settings.condemned_generation = generation_to_condemn (n,
16651 &blocking_collection,
16652 &elevation_requested,
16654 should_evaluate_elevation = elevation_requested;
16655 should_do_blocking_collection = blocking_collection;
16656 #endif //MULTIPLE_HEAPS
16658 settings.condemned_generation = joined_generation_to_condemn (
16659 should_evaluate_elevation,
16660 settings.condemned_generation,
16661 &should_do_blocking_collection
16665 STRESS_LOG1(LF_GCROOTS|LF_GC|LF_GCALLOC, LL_INFO10,
16666 "condemned generation num: %d\n", settings.condemned_generation);
16668 record_gcs_during_no_gc();
16670 if (settings.condemned_generation > 1)
16671 settings.promotion = TRUE;
16673 #ifdef HEAP_ANALYZE
16674 // At this point we've decided what generation is condemned
16675 // See if we've been requested to analyze survivors after the mark phase
16676 if (GCToEEInterface::AnalyzeSurvivorsRequested(settings.condemned_generation))
16678 heap_analyze_enabled = TRUE;
16680 #endif // HEAP_ANALYZE
16682 GCToEEInterface::DiagGCStart(settings.condemned_generation, settings.reason == reason_induced);
16684 #ifdef BACKGROUND_GC
16685 if ((settings.condemned_generation == max_generation) &&
16686 (recursive_gc_sync::background_running_p()))
16688 //TODO BACKGROUND_GC If we just wait for the end of gc, it won't work
16689 // because we have to collect 0 and 1 properly
16690 // in particular, the allocation contexts are gone.
16691 // For now, it is simpler to collect max_generation-1
16692 settings.condemned_generation = max_generation - 1;
16693 dprintf (GTC_LOG, ("bgc - 1 instead of 2"));
16696 if ((settings.condemned_generation == max_generation) &&
16697 (should_do_blocking_collection == FALSE) &&
16698 gc_can_use_concurrent &&
16699 !temp_disable_concurrent_p &&
16700 ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)))
16702 keep_bgc_threads_p = TRUE;
16703 c_write (settings.concurrent, TRUE);
16705 #endif //BACKGROUND_GC
16707 settings.gc_index = (uint32_t)dd_collection_count (dynamic_data_of (0)) + 1;
16709 // Call the EE for start of GC work
16710 // just one thread for MP GC
16711 GCToEEInterface::GcStartWork (settings.condemned_generation,
16714 // TODO: we could fire an ETW event to say this GC as a concurrent GC but later on due to not being able to
16715 // create threads or whatever, this could be a non concurrent GC. Maybe for concurrent GC we should fire
16716 // it in do_background_gc and if it failed to be a CGC we fire it in gc1... in other words, this should be
16720 #ifdef MULTIPLE_HEAPS
16721 gc_start_event.Reset();
16722 //start all threads on the roots.
16723 dprintf(3, ("Starting all gc threads for gc"));
16724 gc_t_join.restart();
16725 #endif //MULTIPLE_HEAPS
16729 int gen_num_for_data = max_generation + 1;
16730 for (int i = 0; i <= gen_num_for_data; i++)
16732 gc_data_per_heap.gen_data[i].size_before = generation_size (i);
16733 generation* gen = generation_of (i);
16734 gc_data_per_heap.gen_data[i].free_list_space_before = generation_free_list_space (gen);
16735 gc_data_per_heap.gen_data[i].free_obj_space_before = generation_free_obj_space (gen);
16738 descr_generations (TRUE);
16739 // descr_card_table();
16742 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
16743 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_POST_GC_ONLY))
16745 verify_heap (TRUE);
16747 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK)
16748 checkGCWriteBarrier();
16750 #endif // VERIFY_HEAP
16752 #ifdef BACKGROUND_GC
16753 if (settings.concurrent)
16755 // We need to save the settings because we'll need to restore it after each FGC.
16756 assert (settings.condemned_generation == max_generation);
16757 settings.compaction = FALSE;
16758 saved_bgc_settings = settings;
16760 #ifdef MULTIPLE_HEAPS
16761 if (heap_number == 0)
16763 for (int i = 0; i < n_heaps; i++)
16765 prepare_bgc_thread (g_heaps[i]);
16767 dprintf (2, ("setting bgc_threads_sync_event"));
16768 bgc_threads_sync_event.Set();
16772 bgc_threads_sync_event.Wait(INFINITE, FALSE);
16773 dprintf (2, ("bgc_threads_sync_event is signalled"));
16776 prepare_bgc_thread(0);
16777 #endif //MULTIPLE_HEAPS
16779 #ifdef MULTIPLE_HEAPS
16780 gc_t_join.join(this, gc_join_start_bgc);
16781 if (gc_t_join.joined())
16782 #endif //MULTIPLE_HEAPS
16784 do_concurrent_p = TRUE;
16785 do_ephemeral_gc_p = FALSE;
16786 #ifdef MULTIPLE_HEAPS
16787 dprintf(2, ("Joined to perform a background GC"));
16789 for (int i = 0; i < n_heaps; i++)
16791 gc_heap* hp = g_heaps[i];
16792 if (!(hp->bgc_thread) || !hp->commit_mark_array_bgc_init (hp->mark_array))
16794 do_concurrent_p = FALSE;
16799 hp->background_saved_lowest_address = hp->lowest_address;
16800 hp->background_saved_highest_address = hp->highest_address;
16804 do_concurrent_p = (!!bgc_thread && commit_mark_array_bgc_init (mark_array));
16805 if (do_concurrent_p)
16807 background_saved_lowest_address = lowest_address;
16808 background_saved_highest_address = highest_address;
16810 #endif //MULTIPLE_HEAPS
16812 if (do_concurrent_p)
16814 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
16815 SoftwareWriteWatch::EnableForGCHeap();
16816 #endif //FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
16818 #ifdef MULTIPLE_HEAPS
16819 for (int i = 0; i < n_heaps; i++)
16820 g_heaps[i]->current_bgc_state = bgc_initialized;
16822 current_bgc_state = bgc_initialized;
16823 #endif //MULTIPLE_HEAPS
16825 int gen = check_for_ephemeral_alloc();
16826 // always do a gen1 GC before we start BGC.
16827 // This is temporary for testing purpose.
16828 //int gen = max_generation - 1;
16829 dont_restart_ee_p = TRUE;
16832 // If we decide to not do a GC before the BGC we need to
16833 // restore the gen0 alloc context.
16834 #ifdef MULTIPLE_HEAPS
16835 for (int i = 0; i < n_heaps; i++)
16837 generation_allocation_pointer (g_heaps[i]->generation_of (0)) = 0;
16838 generation_allocation_limit (g_heaps[i]->generation_of (0)) = 0;
16841 generation_allocation_pointer (youngest_generation) = 0;
16842 generation_allocation_limit (youngest_generation) = 0;
16843 #endif //MULTIPLE_HEAPS
16847 do_ephemeral_gc_p = TRUE;
16849 settings.init_mechanisms();
16850 settings.condemned_generation = gen;
16851 settings.gc_index = (size_t)dd_collection_count (dynamic_data_of (0)) + 2;
16854 // TODO BACKGROUND_GC need to add the profiling stuff here.
16855 dprintf (GTC_LOG, ("doing gen%d before doing a bgc", gen));
16858 //clear the cards so they don't bleed in gen 1 during collection
16859 // shouldn't this always be done at the beginning of any GC?
16860 //clear_card_for_addresses (
16861 // generation_allocation_start (generation_of (0)),
16862 // heap_segment_allocated (ephemeral_heap_segment));
16864 if (!do_ephemeral_gc_p)
16866 do_background_gc();
16871 settings.compaction = TRUE;
16872 c_write (settings.concurrent, FALSE);
16875 #ifdef MULTIPLE_HEAPS
16876 gc_t_join.restart();
16877 #endif //MULTIPLE_HEAPS
16880 if (do_concurrent_p)
16882 // At this point we are sure we'll be starting a BGC, so save its per heap data here.
16883 // global data is only calculated at the end of the GC so we don't need to worry about
16884 // FGCs overwriting it.
16885 memset (&bgc_data_per_heap, 0, sizeof (bgc_data_per_heap));
16886 memcpy (&bgc_data_per_heap, &gc_data_per_heap, sizeof(gc_data_per_heap));
16888 if (do_ephemeral_gc_p)
16890 dprintf (2, ("GC threads running, doing gen%d GC", settings.condemned_generation));
16892 gen_to_condemn_reasons.init();
16893 gen_to_condemn_reasons.set_condition (gen_before_bgc);
16894 gc_data_per_heap.gen_to_condemn_reasons.init (&gen_to_condemn_reasons);
16896 #ifdef MULTIPLE_HEAPS
16897 gc_t_join.join(this, gc_join_bgc_after_ephemeral);
16898 if (gc_t_join.joined())
16899 #endif //MULTIPLE_HEAPS
16901 #ifdef MULTIPLE_HEAPS
16903 #endif //MULTIPLE_HEAPS
16904 settings = saved_bgc_settings;
16905 assert (settings.concurrent);
16907 do_background_gc();
16909 #ifdef MULTIPLE_HEAPS
16910 gc_t_join.restart();
16911 #endif //MULTIPLE_HEAPS
16917 dprintf (2, ("couldn't create BGC threads, reverting to doing a blocking GC"));
16922 #endif //BACKGROUND_GC
16926 #ifndef MULTIPLE_HEAPS
16927 allocation_running_time = (size_t)GCToOSInterface::GetLowPrecisionTimeStamp();
16928 allocation_running_amount = dd_new_allocation (dynamic_data_of (0));
16929 fgn_last_alloc = dd_new_allocation (dynamic_data_of (0));
16930 #endif //MULTIPLE_HEAPS
16933 if (settings.pause_mode == pause_no_gc)
16934 allocate_for_no_gc_after_gc();
16936 int gn = settings.condemned_generation;
16940 #define mark_stack_empty_p() (mark_stack_base == mark_stack_tos)
16943 size_t& gc_heap::promoted_bytes(int thread)
16945 #ifdef MULTIPLE_HEAPS
16946 return g_promoted [thread*16];
16947 #else //MULTIPLE_HEAPS
16948 UNREFERENCED_PARAMETER(thread);
16950 #endif //MULTIPLE_HEAPS
16953 #ifdef INTERIOR_POINTERS
16954 heap_segment* gc_heap::find_segment (uint8_t* interior, BOOL small_segment_only_p)
16956 #ifdef SEG_MAPPING_TABLE
16957 heap_segment* seg = seg_mapping_table_segment_of (interior);
16960 if (small_segment_only_p && heap_segment_loh_p (seg))
16964 #else //SEG_MAPPING_TABLE
16965 #ifdef MULTIPLE_HEAPS
16966 for (int i = 0; i < gc_heap::n_heaps; i++)
16968 gc_heap* h = gc_heap::g_heaps [i];
16969 hs = h->find_segment_per_heap (o, small_segment_only_p);
16977 gc_heap* h = pGenGCHeap;
16978 hs = h->find_segment_per_heap (o, small_segment_only_p);
16980 #endif //MULTIPLE_HEAPS
16981 #endif //SEG_MAPPING_TABLE
16984 heap_segment* gc_heap::find_segment_per_heap (uint8_t* interior, BOOL small_segment_only_p)
16986 #ifdef SEG_MAPPING_TABLE
16987 return find_segment (interior, small_segment_only_p);
16988 #else //SEG_MAPPING_TABLE
16989 if (in_range_for_segment (interior, ephemeral_heap_segment))
16991 return ephemeral_heap_segment;
16995 heap_segment* found_seg = 0;
16998 heap_segment* seg = generation_start_segment (generation_of (max_generation));
17001 if (in_range_for_segment (interior, seg))
17004 goto end_find_segment;
17007 } while ((seg = heap_segment_next (seg)) != 0);
17009 if (!small_segment_only_p)
17011 #ifdef BACKGROUND_GC
17013 ptrdiff_t delta = 0;
17014 heap_segment* seg = segment_of (interior, delta);
17015 if (seg && in_range_for_segment (interior, seg))
17019 goto end_find_segment;
17021 #else //BACKGROUND_GC
17022 heap_segment* seg = generation_start_segment (generation_of (max_generation+1));
17025 if (in_range_for_segment(interior, seg))
17028 goto end_find_segment;
17031 } while ((seg = heap_segment_next (seg)) != 0);
17032 #endif //BACKGROUND_GC
17038 #endif //SEG_MAPPING_TABLE
17040 #endif //INTERIOR_POINTERS
17042 #if !defined(_DEBUG) && !defined(__GNUC__)
17043 inline // This causes link errors if global optimization is off
17044 #endif //!_DEBUG && !__GNUC__
17045 gc_heap* gc_heap::heap_of (uint8_t* o)
17047 #ifdef MULTIPLE_HEAPS
17049 return g_heaps [0];
17050 #ifdef SEG_MAPPING_TABLE
17051 gc_heap* hp = seg_mapping_table_heap_of (o);
17052 return (hp ? hp : g_heaps[0]);
17053 #else //SEG_MAPPING_TABLE
17054 ptrdiff_t delta = 0;
17055 heap_segment* seg = segment_of (o, delta);
17056 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17057 #endif //SEG_MAPPING_TABLE
17058 #else //MULTIPLE_HEAPS
17059 UNREFERENCED_PARAMETER(o);
17061 #endif //MULTIPLE_HEAPS
17065 gc_heap* gc_heap::heap_of_gc (uint8_t* o)
17067 #ifdef MULTIPLE_HEAPS
17069 return g_heaps [0];
17070 #ifdef SEG_MAPPING_TABLE
17071 gc_heap* hp = seg_mapping_table_heap_of_gc (o);
17072 return (hp ? hp : g_heaps[0]);
17073 #else //SEG_MAPPING_TABLE
17074 ptrdiff_t delta = 0;
17075 heap_segment* seg = segment_of (o, delta);
17076 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17077 #endif //SEG_MAPPING_TABLE
17078 #else //MULTIPLE_HEAPS
17079 UNREFERENCED_PARAMETER(o);
17081 #endif //MULTIPLE_HEAPS
17084 #ifdef INTERIOR_POINTERS
17085 // will find all heap objects (large and small)
17086 uint8_t* gc_heap::find_object (uint8_t* interior, uint8_t* low)
17088 if (!gen0_bricks_cleared)
17090 #ifdef MULTIPLE_HEAPS
17091 assert (!"Should have already been done in server GC");
17092 #endif //MULTIPLE_HEAPS
17093 gen0_bricks_cleared = TRUE;
17094 //initialize brick table for gen 0
17095 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
17096 b < brick_of (align_on_brick
17097 (heap_segment_allocated (ephemeral_heap_segment)));
17103 #ifdef FFIND_OBJECT
17104 //indicate that in the future this needs to be done during allocation
17105 #ifdef MULTIPLE_HEAPS
17106 gen0_must_clear_bricks = FFIND_DECAY*gc_heap::n_heaps;
17108 gen0_must_clear_bricks = FFIND_DECAY;
17109 #endif //MULTIPLE_HEAPS
17110 #endif //FFIND_OBJECT
17112 int brick_entry = get_brick_entry(brick_of (interior));
17113 if (brick_entry == 0)
17115 // this is a pointer to a large object
17116 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17118 #ifdef FEATURE_CONSERVATIVE_GC
17119 && (GCConfig::GetConservativeGC() || interior <= heap_segment_allocated(seg))
17123 // If interior falls within the first free object at the beginning of a generation,
17124 // we don't have brick entry for it, and we may incorrectly treat it as on large object heap.
17125 int align_const = get_alignment_constant (heap_segment_read_only_p (seg)
17126 #ifdef FEATURE_CONSERVATIVE_GC
17127 || (GCConfig::GetConservativeGC() && !heap_segment_loh_p (seg))
17130 //int align_const = get_alignment_constant (heap_segment_read_only_p (seg));
17131 assert (interior < heap_segment_allocated (seg));
17133 uint8_t* o = heap_segment_mem (seg);
17134 while (o < heap_segment_allocated (seg))
17136 uint8_t* next_o = o + Align (size (o), align_const);
17137 assert (next_o > o);
17138 if ((o <= interior) && (interior < next_o))
17149 else if (interior >= low)
17151 heap_segment* seg = find_segment_per_heap (interior, TRUE);
17154 #ifdef FEATURE_CONSERVATIVE_GC
17155 if (interior >= heap_segment_allocated (seg))
17158 assert (interior < heap_segment_allocated (seg));
17160 uint8_t* o = find_first_object (interior, heap_segment_mem (seg));
17171 gc_heap::find_object_for_relocation (uint8_t* interior, uint8_t* low, uint8_t* high)
17173 uint8_t* old_address = interior;
17174 if (!((old_address >= low) && (old_address < high)))
17177 size_t brick = brick_of (old_address);
17178 int brick_entry = brick_table [ brick ];
17179 if (brick_entry != 0)
17183 while (brick_entry < 0)
17185 brick = (brick + brick_entry);
17186 brick_entry = brick_table [ brick ];
17188 uint8_t* old_loc = old_address;
17189 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
17191 if (node <= old_loc)
17196 brick_entry = brick_table [ brick ];
17202 //find the object by going along the plug
17204 while (o <= interior)
17206 uint8_t* next_o = o + Align (size (o));
17207 assert (next_o > o);
17208 if (next_o > interior)
17214 assert ((o <= interior) && ((o + Align (size (o))) > interior));
17219 // this is a pointer to a large object
17220 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17223 assert (interior < heap_segment_allocated (seg));
17225 uint8_t* o = heap_segment_mem (seg);
17226 while (o < heap_segment_allocated (seg))
17228 uint8_t* next_o = o + Align (size (o));
17229 assert (next_o > o);
17230 if ((o < interior) && (interior < next_o))
17242 #else //INTERIOR_POINTERS
17244 uint8_t* gc_heap::find_object (uint8_t* o, uint8_t* low)
17248 #endif //INTERIOR_POINTERS
17251 #ifdef GC_CONFIG_DRIVEN
17252 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;} else {mark_list_index++;} if (slow > o) slow = o; if (shigh < o) shigh = o;}
17254 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;}if (slow > o) slow = o; if (shigh < o) shigh = o;}
17255 #endif //GC_CONFIG_DRIVEN
17257 #define m_boundary(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17260 #define m_boundary_fullgc(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17262 #define method_table(o) ((CObjectHeader*)(o))->GetMethodTable()
17265 BOOL gc_heap::gc_mark1 (uint8_t* o)
17267 BOOL marked = !marked (o);
17269 dprintf (3, ("*%Ix*, newly marked: %d", (size_t)o, marked));
17274 BOOL gc_heap::gc_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17276 BOOL marked = FALSE;
17277 if ((o >= low) && (o < high))
17278 marked = gc_mark1 (o);
17279 #ifdef MULTIPLE_HEAPS
17283 gc_heap* hp = heap_of_gc (o);
17285 if ((o >= hp->gc_low) && (o < hp->gc_high))
17286 marked = gc_mark1 (o);
17289 snoop_stat.objects_checked_count++;
17293 snoop_stat.objects_marked_count++;
17297 snoop_stat.zero_ref_count++;
17300 #endif //SNOOP_STATS
17301 #endif //MULTIPLE_HEAPS
17305 #ifdef BACKGROUND_GC
17308 BOOL gc_heap::background_marked (uint8_t* o)
17310 return mark_array_marked (o);
17313 BOOL gc_heap::background_mark1 (uint8_t* o)
17315 BOOL to_mark = !mark_array_marked (o);
17317 dprintf (3, ("b*%Ix*b(%d)", (size_t)o, (to_mark ? 1 : 0)));
17320 mark_array_set_marked (o);
17321 dprintf (4, ("n*%Ix*n", (size_t)o));
17328 // TODO: we could consider filtering out NULL's here instead of going to
17329 // look for it on other heaps
17331 BOOL gc_heap::background_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17333 BOOL marked = FALSE;
17334 if ((o >= low) && (o < high))
17335 marked = background_mark1 (o);
17336 #ifdef MULTIPLE_HEAPS
17340 gc_heap* hp = heap_of (o);
17342 if ((o >= hp->background_saved_lowest_address) && (o < hp->background_saved_highest_address))
17343 marked = background_mark1 (o);
17345 #endif //MULTIPLE_HEAPS
17349 #endif //BACKGROUND_GC
17352 uint8_t* gc_heap::next_end (heap_segment* seg, uint8_t* f)
17354 if (seg == ephemeral_heap_segment)
17357 return heap_segment_allocated (seg);
17360 #define new_start() {if (ppstop <= start) {break;} else {parm = start}}
17361 #define ignore_start 0
17362 #define use_start 1
17364 #define go_through_object(mt,o,size,parm,start,start_useful,limit,exp) \
17366 CGCDesc* map = CGCDesc::GetCGCDescFromMT((MethodTable*)(mt)); \
17367 CGCDescSeries* cur = map->GetHighestSeries(); \
17368 ptrdiff_t cnt = (ptrdiff_t) map->GetNumSeries(); \
17372 CGCDescSeries* last = map->GetLowestSeries(); \
17373 uint8_t** parm = 0; \
17376 assert (parm <= (uint8_t**)((o) + cur->GetSeriesOffset())); \
17377 parm = (uint8_t**)((o) + cur->GetSeriesOffset()); \
17378 uint8_t** ppstop = \
17379 (uint8_t**)((uint8_t*)parm + cur->GetSeriesSize() + (size));\
17380 if (!start_useful || (uint8_t*)ppstop > (start)) \
17382 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start);\
17383 while (parm < ppstop) \
17391 } while (cur >= last); \
17395 /* Handle the repeating case - array of valuetypes */ \
17396 uint8_t** parm = (uint8_t**)((o) + cur->startoffset); \
17397 if (start_useful && start > (uint8_t*)parm) \
17399 ptrdiff_t cs = mt->RawGetComponentSize(); \
17400 parm = (uint8_t**)((uint8_t*)parm + (((start) - (uint8_t*)parm)/cs)*cs); \
17402 while ((uint8_t*)parm < ((o)+(size)-plug_skew)) \
17404 for (ptrdiff_t __i = 0; __i > cnt; __i--) \
17406 HALF_SIZE_T skip = cur->val_serie[__i].skip; \
17407 HALF_SIZE_T nptrs = cur->val_serie[__i].nptrs; \
17408 uint8_t** ppstop = parm + nptrs; \
17409 if (!start_useful || (uint8_t*)ppstop > (start)) \
17411 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start); \
17416 } while (parm < ppstop); \
17418 parm = (uint8_t**)((uint8_t*)ppstop + skip); \
17424 #define go_through_object_nostart(mt,o,size,parm,exp) {go_through_object(mt,o,size,parm,o,ignore_start,(o + size),exp); }
17426 // 1 thing to note about this macro:
17427 // 1) you can use *parm safely but in general you don't want to use parm
17428 // because for the collectible types it's not an address on the managed heap.
17429 #ifndef COLLECTIBLE_CLASS
17430 #define go_through_object_cl(mt,o,size,parm,exp) \
17432 if (header(o)->ContainsPointers()) \
17434 go_through_object_nostart(mt,o,size,parm,exp); \
17437 #else //COLLECTIBLE_CLASS
17438 #define go_through_object_cl(mt,o,size,parm,exp) \
17440 if (header(o)->Collectible()) \
17442 uint8_t* class_obj = get_class_object (o); \
17443 uint8_t** parm = &class_obj; \
17444 do {exp} while (false); \
17446 if (header(o)->ContainsPointers()) \
17448 go_through_object_nostart(mt,o,size,parm,exp); \
17451 #endif //COLLECTIBLE_CLASS
17453 // This starts a plug. But mark_stack_tos isn't increased until set_pinned_info is called.
17454 void gc_heap::enque_pinned_plug (uint8_t* plug,
17455 BOOL save_pre_plug_info_p,
17456 uint8_t* last_object_in_last_plug)
17458 if (mark_stack_array_length <= mark_stack_tos)
17460 if (!grow_mark_stack (mark_stack_array, mark_stack_array_length, MARK_STACK_INITIAL_LENGTH))
17462 // we don't want to continue here due to security
17463 // risks. This happens very rarely and fixing it in the
17464 // way so that we can continue is a bit involved and will
17465 // not be done in Dev10.
17466 GCToEEInterface::HandleFatalError(CORINFO_EXCEPTION_GC);
17470 dprintf (3, ("enqueuing P #%Id(%Ix): %Ix. oldest: %Id, LO: %Ix, pre: %d",
17471 mark_stack_tos, &mark_stack_array[mark_stack_tos], plug, mark_stack_bos, last_object_in_last_plug, (save_pre_plug_info_p ? 1 : 0)));
17472 mark& m = mark_stack_array[mark_stack_tos];
17474 // Must be set now because if we have a short object we'll need the value of saved_pre_p.
17475 m.saved_pre_p = save_pre_plug_info_p;
17477 if (save_pre_plug_info_p)
17480 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17482 clear_plug_padded (last_object_in_last_plug);
17483 #endif //SHORT_PLUGS
17484 memcpy (&(m.saved_pre_plug), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17487 set_plug_padded (last_object_in_last_plug);
17488 #endif //SHORT_PLUGS
17490 memcpy (&(m.saved_pre_plug_reloc), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17492 // If the last object in the last plug is too short, it requires special handling.
17493 size_t last_obj_size = plug - last_object_in_last_plug;
17494 if (last_obj_size < min_pre_pin_obj_size)
17496 record_interesting_data_point (idp_pre_short);
17499 record_interesting_data_point (idp_pre_short_padded);
17500 #endif //SHORT_PLUGS
17501 dprintf (3, ("encountered a short object %Ix right before pinned plug %Ix!",
17502 last_object_in_last_plug, plug));
17503 // Need to set the short bit regardless of having refs or not because we need to
17504 // indicate that this object is not walkable.
17507 #ifdef COLLECTIBLE_CLASS
17508 if (is_collectible (last_object_in_last_plug))
17510 m.set_pre_short_collectible();
17512 #endif //COLLECTIBLE_CLASS
17514 if (contain_pointers (last_object_in_last_plug))
17516 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
17518 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
17520 size_t gap_offset = (((size_t)pval - (size_t)(plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
17521 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
17522 m.set_pre_short_bit (gap_offset);
17529 m.saved_post_p = FALSE;
17532 void gc_heap::save_post_plug_info (uint8_t* last_pinned_plug, uint8_t* last_object_in_last_plug, uint8_t* post_plug)
17534 UNREFERENCED_PARAMETER(last_pinned_plug);
17536 mark& m = mark_stack_array[mark_stack_tos - 1];
17537 assert (last_pinned_plug == m.first);
17538 m.saved_post_plug_info_start = (uint8_t*)&(((plug_and_gap*)post_plug)[-1]);
17541 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17543 clear_plug_padded (last_object_in_last_plug);
17544 #endif //SHORT_PLUGS
17545 memcpy (&(m.saved_post_plug), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17548 set_plug_padded (last_object_in_last_plug);
17549 #endif //SHORT_PLUGS
17551 memcpy (&(m.saved_post_plug_reloc), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17553 // This is important - we need to clear all bits here except the last one.
17554 m.saved_post_p = TRUE;
17557 m.saved_post_plug_debug.gap = 1;
17560 dprintf (3, ("PP %Ix has NP %Ix right after", last_pinned_plug, post_plug));
17562 size_t last_obj_size = post_plug - last_object_in_last_plug;
17563 if (last_obj_size < min_pre_pin_obj_size)
17565 dprintf (3, ("PP %Ix last obj %Ix is too short", last_pinned_plug, last_object_in_last_plug));
17566 record_interesting_data_point (idp_post_short);
17569 record_interesting_data_point (idp_post_short_padded);
17570 #endif //SHORT_PLUGS
17571 m.set_post_short();
17572 verify_pinned_queue_p = TRUE;
17574 #ifdef COLLECTIBLE_CLASS
17575 if (is_collectible (last_object_in_last_plug))
17577 m.set_post_short_collectible();
17579 #endif //COLLECTIBLE_CLASS
17581 if (contain_pointers (last_object_in_last_plug))
17583 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
17585 // TODO: since we won't be able to walk this object in relocation, we still need to
17586 // take care of collectible assemblies here.
17587 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
17589 size_t gap_offset = (((size_t)pval - (size_t)(post_plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
17590 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
17591 m.set_post_short_bit (gap_offset);
17600 __declspec(naked) void __fastcall Prefetch(void* addr)
17608 inline void Prefetch (void* addr)
17610 UNREFERENCED_PARAMETER(addr);
17615 VOLATILE(uint8_t*)& gc_heap::ref_mark_stack (gc_heap* hp, int index)
17617 return ((VOLATILE(uint8_t*)*)(hp->mark_stack_array))[index];
17620 #endif //MH_SC_MARK
17624 #define partial_object 3
17626 uint8_t* ref_from_slot (uint8_t* r)
17628 return (uint8_t*)((size_t)r & ~(stolen | partial));
17631 BOOL stolen_p (uint8_t* r)
17633 return (((size_t)r&2) && !((size_t)r&1));
17636 BOOL ready_p (uint8_t* r)
17638 return ((size_t)r != 1);
17641 BOOL partial_p (uint8_t* r)
17643 return (((size_t)r&1) && !((size_t)r&2));
17646 BOOL straight_ref_p (uint8_t* r)
17648 return (!stolen_p (r) && !partial_p (r));
17651 BOOL partial_object_p (uint8_t* r)
17653 return (((size_t)r & partial_object) == partial_object);
17656 BOOL ref_p (uint8_t* r)
17658 return (straight_ref_p (r) || partial_object_p (r));
17661 void gc_heap::mark_object_simple1 (uint8_t* oo, uint8_t* start THREAD_NUMBER_DCL)
17663 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_tos = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)mark_stack_array;
17664 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_limit = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)&mark_stack_array[mark_stack_array_length];
17665 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_base = mark_stack_tos;
17666 #ifdef SORT_MARK_STACK
17667 SERVER_SC_MARK_VOLATILE(uint8_t*)* sorted_tos = mark_stack_base;
17668 #endif //SORT_MARK_STACK
17670 // If we are doing a full GC we don't use mark list anyway so use m_boundary_fullgc that doesn't
17671 // update mark list.
17672 BOOL full_p = (settings.condemned_generation == max_generation);
17674 assert ((start >= oo) && (start < oo+size(oo)));
17677 *mark_stack_tos = oo;
17678 #endif //!MH_SC_MARK
17682 #ifdef MULTIPLE_HEAPS
17683 #else //MULTIPLE_HEAPS
17684 const int thread = 0;
17685 #endif //MULTIPLE_HEAPS
17687 if (oo && ((size_t)oo != 4))
17695 else if (!partial_p (oo) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
17697 BOOL overflow_p = FALSE;
17699 if (mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
17701 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
17702 if (mark_stack_tos + CGCDesc::GetNumPointers(method_table(oo), s, num_components) >= (mark_stack_limit - 1))
17708 if (overflow_p == FALSE)
17710 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
17712 go_through_object_cl (method_table(oo), oo, s, ppslot,
17714 uint8_t* o = *ppslot;
17716 if (gc_mark (o, gc_low, gc_high))
17720 m_boundary_fullgc (o);
17726 size_t obj_size = size (o);
17727 promoted_bytes (thread) += obj_size;
17728 if (contain_pointers_or_collectible (o))
17730 *(mark_stack_tos++) = o;
17738 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
17739 min_overflow_address = min (min_overflow_address, oo);
17740 max_overflow_address = max (max_overflow_address, oo);
17745 if (partial_p (oo))
17747 start = ref_from_slot (oo);
17748 oo = ref_from_slot (*(--mark_stack_tos));
17749 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
17750 assert ((oo < start) && (start < (oo + size (oo))));
17752 #ifdef COLLECTIBLE_CLASS
17755 // If there's a class object, push it now. We are guaranteed to have the slot since
17756 // we just popped one object off.
17757 if (is_collectible (oo))
17759 uint8_t* class_obj = get_class_object (oo);
17760 if (gc_mark (class_obj, gc_low, gc_high))
17764 m_boundary_fullgc (class_obj);
17768 m_boundary (class_obj);
17771 size_t obj_size = size (class_obj);
17772 promoted_bytes (thread) += obj_size;
17773 *(mark_stack_tos++) = class_obj;
17777 #endif //COLLECTIBLE_CLASS
17781 BOOL overflow_p = FALSE;
17783 if (mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
17787 if (overflow_p == FALSE)
17789 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
17791 //push the object and its current
17792 SERVER_SC_MARK_VOLATILE(uint8_t*)* place = ++mark_stack_tos;
17796 *(place) = (uint8_t*)partial;
17797 #endif //MH_SC_MARK
17798 int i = num_partial_refs;
17799 uint8_t* ref_to_continue = 0;
17801 go_through_object (method_table(oo), oo, s, ppslot,
17802 start, use_start, (oo + s),
17804 uint8_t* o = *ppslot;
17806 if (gc_mark (o, gc_low, gc_high))
17810 m_boundary_fullgc (o);
17816 size_t obj_size = size (o);
17817 promoted_bytes (thread) += obj_size;
17818 if (contain_pointers_or_collectible (o))
17820 *(mark_stack_tos++) = o;
17823 ref_to_continue = (uint8_t*)((size_t)(ppslot+1) | partial);
17832 //we are finished with this object
17833 assert (ref_to_continue == 0);
17835 assert ((*(place-1)) == (uint8_t*)0);
17838 #endif //MH_SC_MARK
17840 // shouldn't we decrease tos by 2 here??
17843 if (ref_to_continue)
17847 assert ((*(place-1)) == (uint8_t*)0);
17848 *(place-1) = (uint8_t*)((size_t)oo | partial_object);
17849 assert (((*place) == (uint8_t*)1) || ((*place) == (uint8_t*)2));
17850 #endif //MH_SC_MARK
17851 *place = ref_to_continue;
17856 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
17857 min_overflow_address = min (min_overflow_address, oo);
17858 max_overflow_address = max (max_overflow_address, oo);
17861 #ifdef SORT_MARK_STACK
17862 if (mark_stack_tos > sorted_tos + mark_stack_array_length/8)
17864 rqsort1 (sorted_tos, mark_stack_tos-1);
17865 sorted_tos = mark_stack_tos-1;
17867 #endif //SORT_MARK_STACK
17870 if (!(mark_stack_empty_p()))
17872 oo = *(--mark_stack_tos);
17875 #ifdef SORT_MARK_STACK
17876 sorted_tos = min ((size_t)sorted_tos, (size_t)mark_stack_tos);
17877 #endif //SORT_MARK_STACK
17885 BOOL same_numa_node_p (int hn1, int hn2)
17887 return (heap_select::find_numa_node_from_heap_no (hn1) == heap_select::find_numa_node_from_heap_no (hn2));
17890 int find_next_buddy_heap (int this_heap_number, int current_buddy, int n_heaps)
17892 int hn = (current_buddy+1)%n_heaps;
17893 while (hn != current_buddy)
17895 if ((this_heap_number != hn) && (same_numa_node_p (this_heap_number, hn)))
17897 hn = (hn+1)%n_heaps;
17899 return current_buddy;
17903 gc_heap::mark_steal()
17905 mark_stack_busy() = 0;
17906 //clear the mark stack in the snooping range
17907 for (int i = 0; i < max_snoop_level; i++)
17909 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
17912 //pick the next heap as our buddy
17913 int thpn = find_next_buddy_heap (heap_number, heap_number, n_heaps);
17916 dprintf (SNOOP_LOG, ("(GC%d)heap%d: start snooping %d", settings.gc_index, heap_number, (heap_number+1)%n_heaps));
17917 uint32_t begin_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
17918 #endif //SNOOP_STATS
17920 int idle_loop_count = 0;
17921 int first_not_ready_level = 0;
17925 gc_heap* hp = g_heaps [thpn];
17926 int level = first_not_ready_level;
17927 first_not_ready_level = 0;
17929 while (check_next_mark_stack (hp) && (level < (max_snoop_level-1)))
17931 idle_loop_count = 0;
17933 snoop_stat.busy_count++;
17934 dprintf (SNOOP_LOG, ("heap%d: looking at next heap level %d stack contents: %Ix",
17935 heap_number, level, (int)((uint8_t**)(hp->mark_stack_array))[level]));
17936 #endif //SNOOP_STATS
17938 uint8_t* o = ref_mark_stack (hp, level);
17940 uint8_t* start = o;
17943 mark_stack_busy() = 1;
17945 BOOL success = TRUE;
17946 uint8_t* next = (ref_mark_stack (hp, level+1));
17949 if (((size_t)o > 4) && !partial_object_p (o))
17951 //this is a normal object, not a partial mark tuple
17952 //success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), 0, o)==o);
17953 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), (uint8_t*)4, o)==o);
17955 snoop_stat.interlocked_count++;
17957 snoop_stat.normal_count++;
17958 #endif //SNOOP_STATS
17962 //it is a stolen entry, or beginning/ending of a partial mark
17965 snoop_stat.stolen_or_pm_count++;
17966 #endif //SNOOP_STATS
17970 else if (stolen_p (next))
17972 //ignore the stolen guy and go to the next level
17976 snoop_stat.stolen_entry_count++;
17977 #endif //SNOOP_STATS
17981 assert (partial_p (next));
17982 start = ref_from_slot (next);
17983 //re-read the object
17984 o = ref_from_slot (ref_mark_stack (hp, level));
17988 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level+1), (uint8_t*)stolen, next)==next);
17990 snoop_stat.interlocked_count++;
17993 snoop_stat.partial_mark_parent_count++;
17995 #endif //SNOOP_STATS
17999 // stack is not ready, or o is completely different from the last time we read from this stack level.
18000 // go up 2 levels to steal children or totally unrelated objects.
18002 if (first_not_ready_level == 0)
18004 first_not_ready_level = level;
18008 snoop_stat.pm_not_ready_count++;
18009 #endif //SNOOP_STATS
18016 dprintf (SNOOP_LOG, ("heap%d: marking %Ix from %d [%d] tl:%dms",
18017 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18018 (GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18019 uint32_t start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
18020 #endif //SNOOP_STATS
18022 mark_object_simple1 (o, start, heap_number);
18025 dprintf (SNOOP_LOG, ("heap%d: done marking %Ix from %d [%d] %dms tl:%dms",
18026 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18027 (GCToOSInterface::GetLowPrecisionTimeStamp()-start_tick),(GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18028 #endif //SNOOP_STATS
18030 mark_stack_busy() = 0;
18032 //clear the mark stack in snooping range
18033 for (int i = 0; i < max_snoop_level; i++)
18035 if (((uint8_t**)mark_stack_array)[i] != 0)
18037 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
18039 snoop_stat.stack_bottom_clear_count++;
18040 #endif //SNOOP_STATS
18046 mark_stack_busy() = 0;
18050 //slot is either partial or stolen
18054 if ((first_not_ready_level != 0) && hp->mark_stack_busy())
18058 if (!hp->mark_stack_busy())
18060 first_not_ready_level = 0;
18063 if ((idle_loop_count % (6) )==1)
18066 snoop_stat.switch_to_thread_count++;
18067 #endif //SNOOP_STATS
18068 GCToOSInterface::Sleep(1);
18070 int free_count = 1;
18072 snoop_stat.stack_idle_count++;
18073 //dprintf (SNOOP_LOG, ("heap%d: counting idle threads", heap_number));
18074 #endif //SNOOP_STATS
18075 for (int hpn = (heap_number+1)%n_heaps; hpn != heap_number;)
18077 if (!((g_heaps [hpn])->mark_stack_busy()))
18081 dprintf (SNOOP_LOG, ("heap%d: %d idle", heap_number, free_count));
18082 #endif //SNOOP_STATS
18084 else if (same_numa_node_p (hpn, heap_number) || ((idle_loop_count%1000))==999)
18089 hpn = (hpn+1)%n_heaps;
18092 if (free_count == n_heaps)
18101 BOOL gc_heap::check_next_mark_stack (gc_heap* next_heap)
18104 snoop_stat.check_level_count++;
18105 #endif //SNOOP_STATS
18106 return (next_heap->mark_stack_busy()>=1);
18108 #endif //MH_SC_MARK
18111 void gc_heap::print_snoop_stat()
18113 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18114 "heap", "check", "zero", "mark", "stole", "pstack", "nstack", "nonsk"));
18115 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d",
18116 snoop_stat.heap_index,
18117 snoop_stat.objects_checked_count,
18118 snoop_stat.zero_ref_count,
18119 snoop_stat.objects_marked_count,
18120 snoop_stat.stolen_stack_count,
18121 snoop_stat.partial_stack_count,
18122 snoop_stat.normal_stack_count,
18123 snoop_stat.non_stack_count));
18124 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18125 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "clear"));
18126 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18127 snoop_stat.heap_index,
18128 snoop_stat.check_level_count,
18129 snoop_stat.busy_count,
18130 snoop_stat.interlocked_count,
18131 snoop_stat.partial_mark_parent_count,
18132 snoop_stat.stolen_or_pm_count,
18133 snoop_stat.stolen_entry_count,
18134 snoop_stat.pm_not_ready_count,
18135 snoop_stat.normal_count,
18136 snoop_stat.stack_bottom_clear_count));
18138 printf ("\n%4s | %8s | %8s | %8s | %8s | %8s\n",
18139 "heap", "check", "zero", "mark", "idle", "switch");
18140 printf ("%4d | %8d | %8d | %8d | %8d | %8d\n",
18141 snoop_stat.heap_index,
18142 snoop_stat.objects_checked_count,
18143 snoop_stat.zero_ref_count,
18144 snoop_stat.objects_marked_count,
18145 snoop_stat.stack_idle_count,
18146 snoop_stat.switch_to_thread_count);
18147 printf ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
18148 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
18149 printf ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18150 snoop_stat.heap_index,
18151 snoop_stat.check_level_count,
18152 snoop_stat.busy_count,
18153 snoop_stat.interlocked_count,
18154 snoop_stat.partial_mark_parent_count,
18155 snoop_stat.stolen_or_pm_count,
18156 snoop_stat.stolen_entry_count,
18157 snoop_stat.pm_not_ready_count,
18158 snoop_stat.normal_count,
18159 snoop_stat.stack_bottom_clear_count);
18161 #endif //SNOOP_STATS
18163 #ifdef HEAP_ANALYZE
18165 gc_heap::ha_mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18167 if (!internal_root_array)
18169 internal_root_array = new (nothrow) uint8_t* [internal_root_array_length];
18170 if (!internal_root_array)
18172 heap_analyze_success = FALSE;
18176 if (heap_analyze_success && (internal_root_array_length <= internal_root_array_index))
18178 size_t new_size = 2*internal_root_array_length;
18180 uint64_t available_physical = 0;
18181 get_memory_info (NULL, &available_physical);
18182 if (new_size > (size_t)(available_physical / 10))
18184 heap_analyze_success = FALSE;
18188 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
18191 memcpy (tmp, internal_root_array,
18192 internal_root_array_length*sizeof (uint8_t*));
18193 delete[] internal_root_array;
18194 internal_root_array = tmp;
18195 internal_root_array_length = new_size;
18199 heap_analyze_success = FALSE;
18204 if (heap_analyze_success)
18206 PREFIX_ASSUME(internal_root_array_index < internal_root_array_length);
18208 uint8_t* ref = (uint8_t*)po;
18209 if (!current_obj ||
18210 !((ref >= current_obj) && (ref < (current_obj + current_obj_size))))
18212 gc_heap* hp = gc_heap::heap_of (ref);
18213 current_obj = hp->find_object (ref, hp->lowest_address);
18214 current_obj_size = size (current_obj);
18216 internal_root_array[internal_root_array_index] = current_obj;
18217 internal_root_array_index++;
18221 mark_object_simple (po THREAD_NUMBER_ARG);
18223 #endif //HEAP_ANALYZE
18225 //this method assumes that *po is in the [low. high[ range
18227 gc_heap::mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18230 #ifdef MULTIPLE_HEAPS
18231 #else //MULTIPLE_HEAPS
18232 const int thread = 0;
18233 #endif //MULTIPLE_HEAPS
18236 snoop_stat.objects_checked_count++;
18237 #endif //SNOOP_STATS
18242 size_t s = size (o);
18243 promoted_bytes (thread) += s;
18245 go_through_object_cl (method_table(o), o, s, poo,
18247 uint8_t* oo = *poo;
18248 if (gc_mark (oo, gc_low, gc_high))
18251 size_t obj_size = size (oo);
18252 promoted_bytes (thread) += obj_size;
18254 if (contain_pointers_or_collectible (oo))
18255 mark_object_simple1 (oo, oo THREAD_NUMBER_ARG);
18265 uint8_t* gc_heap::mark_object (uint8_t* o THREAD_NUMBER_DCL)
18267 if ((o >= gc_low) && (o < gc_high))
18268 mark_object_simple (&o THREAD_NUMBER_ARG);
18269 #ifdef MULTIPLE_HEAPS
18273 gc_heap* hp = heap_of (o);
18275 if ((o >= hp->gc_low) && (o < hp->gc_high))
18276 mark_object_simple (&o THREAD_NUMBER_ARG);
18278 #endif //MULTIPLE_HEAPS
18283 #ifdef BACKGROUND_GC
18285 void gc_heap::background_mark_simple1 (uint8_t* oo THREAD_NUMBER_DCL)
18287 uint8_t** mark_stack_limit = &background_mark_stack_array[background_mark_stack_array_length];
18289 #ifdef SORT_MARK_STACK
18290 uint8_t** sorted_tos = background_mark_stack_array;
18291 #endif //SORT_MARK_STACK
18293 background_mark_stack_tos = background_mark_stack_array;
18297 #ifdef MULTIPLE_HEAPS
18298 #else //MULTIPLE_HEAPS
18299 const int thread = 0;
18300 #endif //MULTIPLE_HEAPS
18304 if ((((size_t)oo & 1) == 0) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
18306 BOOL overflow_p = FALSE;
18308 if (background_mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
18310 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18311 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18312 if (background_mark_stack_tos + num_pointers >= (mark_stack_limit - 1))
18314 dprintf (2, ("h%d: %Id left, obj (mt: %Ix) %Id ptrs",
18316 (size_t)(mark_stack_limit - 1 - background_mark_stack_tos),
18320 bgc_overflow_count++;
18325 if (overflow_p == FALSE)
18327 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18329 go_through_object_cl (method_table(oo), oo, s, ppslot,
18331 uint8_t* o = *ppslot;
18333 if (background_mark (o,
18334 background_saved_lowest_address,
18335 background_saved_highest_address))
18338 size_t obj_size = size (o);
18339 bpromoted_bytes (thread) += obj_size;
18340 if (contain_pointers_or_collectible (o))
18342 *(background_mark_stack_tos++) = o;
18351 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18352 background_min_overflow_address = min (background_min_overflow_address, oo);
18353 background_max_overflow_address = max (background_max_overflow_address, oo);
18358 uint8_t* start = oo;
18359 if ((size_t)oo & 1)
18361 oo = (uint8_t*)((size_t)oo & ~1);
18362 start = *(--background_mark_stack_tos);
18363 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
18365 #ifdef COLLECTIBLE_CLASS
18368 // If there's a class object, push it now. We are guaranteed to have the slot since
18369 // we just popped one object off.
18370 if (is_collectible (oo))
18372 uint8_t* class_obj = get_class_object (oo);
18373 if (background_mark (class_obj,
18374 background_saved_lowest_address,
18375 background_saved_highest_address))
18377 size_t obj_size = size (class_obj);
18378 bpromoted_bytes (thread) += obj_size;
18380 *(background_mark_stack_tos++) = class_obj;
18384 #endif //COLLECTIBLE_CLASS
18388 BOOL overflow_p = FALSE;
18390 if (background_mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
18392 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18393 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18395 dprintf (2, ("h%d: PM: %Id left, obj %Ix (mt: %Ix) start: %Ix, total: %Id",
18397 (size_t)(mark_stack_limit - background_mark_stack_tos),
18403 bgc_overflow_count++;
18406 if (overflow_p == FALSE)
18408 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18410 //push the object and its current
18411 uint8_t** place = background_mark_stack_tos++;
18413 *(background_mark_stack_tos++) = (uint8_t*)((size_t)oo | 1);
18415 int i = num_partial_refs;
18417 go_through_object (method_table(oo), oo, s, ppslot,
18418 start, use_start, (oo + s),
18420 uint8_t* o = *ppslot;
18423 if (background_mark (o,
18424 background_saved_lowest_address,
18425 background_saved_highest_address))
18428 size_t obj_size = size (o);
18429 bpromoted_bytes (thread) += obj_size;
18430 if (contain_pointers_or_collectible (o))
18432 *(background_mark_stack_tos++) = o;
18436 *place = (uint8_t*)(ppslot+1);
18445 //we are finished with this object
18453 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18454 background_min_overflow_address = min (background_min_overflow_address, oo);
18455 background_max_overflow_address = max (background_max_overflow_address, oo);
18459 #ifdef SORT_MARK_STACK
18460 if (background_mark_stack_tos > sorted_tos + mark_stack_array_length/8)
18462 rqsort1 (sorted_tos, background_mark_stack_tos-1);
18463 sorted_tos = background_mark_stack_tos-1;
18465 #endif //SORT_MARK_STACK
18469 if (!(background_mark_stack_tos == background_mark_stack_array))
18471 oo = *(--background_mark_stack_tos);
18473 #ifdef SORT_MARK_STACK
18474 sorted_tos = (uint8_t**)min ((size_t)sorted_tos, (size_t)background_mark_stack_tos);
18475 #endif //SORT_MARK_STACK
18481 assert (background_mark_stack_tos == background_mark_stack_array);
18486 //this version is different than the foreground GC because
18487 //it can't keep pointers to the inside of an object
18488 //while calling background_mark_simple1. The object could be moved
18489 //by an intervening foreground gc.
18490 //this method assumes that *po is in the [low. high[ range
18492 gc_heap::background_mark_simple (uint8_t* o THREAD_NUMBER_DCL)
18494 #ifdef MULTIPLE_HEAPS
18495 #else //MULTIPLE_HEAPS
18496 const int thread = 0;
18497 #endif //MULTIPLE_HEAPS
18499 dprintf (3, ("bmarking %Ix", o));
18501 if (background_mark1 (o))
18504 size_t s = size (o);
18505 bpromoted_bytes (thread) += s;
18507 if (contain_pointers_or_collectible (o))
18509 background_mark_simple1 (o THREAD_NUMBER_ARG);
18516 uint8_t* gc_heap::background_mark_object (uint8_t* o THREAD_NUMBER_DCL)
18518 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
18520 background_mark_simple (o THREAD_NUMBER_ARG);
18526 dprintf (3, ("or-%Ix", o));
18532 void gc_heap::background_verify_mark (Object*& object, ScanContext* sc, uint32_t flags)
18534 UNREFERENCED_PARAMETER(sc);
18536 assert (settings.concurrent);
18537 uint8_t* o = (uint8_t*)object;
18539 gc_heap* hp = gc_heap::heap_of (o);
18540 #ifdef INTERIOR_POINTERS
18541 if (flags & GC_CALL_INTERIOR)
18543 o = hp->find_object (o, background_saved_lowest_address);
18545 #endif //INTERIOR_POINTERS
18547 if (!background_object_marked (o, FALSE))
18553 void gc_heap::background_promote (Object** ppObject, ScanContext* sc, uint32_t flags)
18555 UNREFERENCED_PARAMETER(sc);
18556 //in order to save space on the array, mark the object,
18557 //knowing that it will be visited later
18558 assert (settings.concurrent);
18560 THREAD_NUMBER_FROM_CONTEXT;
18561 #ifndef MULTIPLE_HEAPS
18562 const int thread = 0;
18563 #endif //!MULTIPLE_HEAPS
18565 uint8_t* o = (uint8_t*)*ppObject;
18570 #ifdef DEBUG_DestroyedHandleValue
18571 // we can race with destroy handle during concurrent scan
18572 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
18574 #endif //DEBUG_DestroyedHandleValue
18578 gc_heap* hp = gc_heap::heap_of (o);
18580 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
18585 #ifdef INTERIOR_POINTERS
18586 if (flags & GC_CALL_INTERIOR)
18588 o = hp->find_object (o, hp->background_saved_lowest_address);
18592 #endif //INTERIOR_POINTERS
18594 #ifdef FEATURE_CONSERVATIVE_GC
18595 // For conservative GC, a value on stack may point to middle of a free object.
18596 // In this case, we don't need to promote the pointer.
18597 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
18601 #endif //FEATURE_CONSERVATIVE_GC
18604 ((CObjectHeader*)o)->Validate();
18607 dprintf (BGC_LOG, ("Background Promote %Ix", (size_t)o));
18609 //needs to be called before the marking because it is possible for a foreground
18610 //gc to take place during the mark and move the object
18611 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
18613 hpt->background_mark_simple (o THREAD_NUMBER_ARG);
18616 //used by the ephemeral collection to scan the local background structures
18617 //containing references.
18619 gc_heap::scan_background_roots (promote_func* fn, int hn, ScanContext *pSC)
18625 pSC->thread_number = hn;
18627 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
18628 pSC->pCurrentDomain = 0;
18631 BOOL relocate_p = (fn == &GCHeap::Relocate);
18633 dprintf (3, ("Scanning background mark list"));
18636 size_t mark_list_finger = 0;
18637 while (mark_list_finger < c_mark_list_index)
18639 uint8_t** o = &c_mark_list [mark_list_finger];
18642 // We may not be able to calculate the size during relocate as POPO
18643 // may have written over the object.
18644 size_t s = size (*o);
18645 assert (Align (s) >= Align (min_obj_size));
18646 dprintf(3,("background root %Ix", (size_t)*o));
18648 (*fn) ((Object**)o, pSC, 0);
18649 mark_list_finger++;
18652 //scan the mark stack
18653 dprintf (3, ("Scanning background mark stack"));
18655 uint8_t** finger = background_mark_stack_array;
18656 while (finger < background_mark_stack_tos)
18658 if ((finger + 1) < background_mark_stack_tos)
18660 // We need to check for the partial mark case here.
18661 uint8_t* parent_obj = *(finger + 1);
18662 if ((size_t)parent_obj & 1)
18664 uint8_t* place = *finger;
18665 size_t place_offset = 0;
18666 uint8_t* real_parent_obj = (uint8_t*)((size_t)parent_obj & ~1);
18670 *(finger + 1) = real_parent_obj;
18671 place_offset = place - real_parent_obj;
18672 dprintf(3,("relocating background root %Ix", (size_t)real_parent_obj));
18673 (*fn) ((Object**)(finger + 1), pSC, 0);
18674 real_parent_obj = *(finger + 1);
18675 *finger = real_parent_obj + place_offset;
18676 *(finger + 1) = (uint8_t*)((size_t)real_parent_obj | 1);
18677 dprintf(3,("roots changed to %Ix, %Ix", *finger, *(finger + 1)));
18681 uint8_t** temp = &real_parent_obj;
18682 dprintf(3,("marking background root %Ix", (size_t)real_parent_obj));
18683 (*fn) ((Object**)temp, pSC, 0);
18690 dprintf(3,("background root %Ix", (size_t)*finger));
18691 (*fn) ((Object**)finger, pSC, 0);
18697 void gc_heap::background_mark_through_object (uint8_t* oo THREAD_NUMBER_DCL)
18699 if (contain_pointers (oo))
18701 size_t total_refs = 0;
18702 size_t s = size (oo);
18703 go_through_object_nostart (method_table(oo), oo, s, po,
18707 background_mark_object (o THREAD_NUMBER_ARG);
18711 dprintf (3,("Background marking through %Ix went through %Id refs",
18717 uint8_t* gc_heap::background_seg_end (heap_segment* seg, BOOL concurrent_p)
18719 if (concurrent_p && (seg == saved_overflow_ephemeral_seg))
18721 // for now we stop at where gen1 started when we started processing
18722 return background_min_soh_overflow_address;
18726 return heap_segment_allocated (seg);
18730 uint8_t* gc_heap::background_first_overflow (uint8_t* min_add,
18733 BOOL small_object_p)
18737 if (small_object_p)
18739 if (in_range_for_segment (min_add, seg))
18741 // min_add was the beginning of gen1 when we did the concurrent
18742 // overflow. Now we could be in a situation where min_add is
18743 // actually the same as allocated for that segment (because
18744 // we expanded heap), in which case we can not call
18745 // find first on this address or we will AV.
18746 if (min_add >= heap_segment_allocated (seg))
18752 if (concurrent_p &&
18753 ((seg == saved_overflow_ephemeral_seg) && (min_add >= background_min_soh_overflow_address)))
18755 return background_min_soh_overflow_address;
18759 o = find_first_object (min_add, heap_segment_mem (seg));
18766 o = max (heap_segment_mem (seg), min_add);
18770 void gc_heap::background_process_mark_overflow_internal (int condemned_gen_number,
18771 uint8_t* min_add, uint8_t* max_add,
18776 current_bgc_state = bgc_overflow_soh;
18779 size_t total_marked_objects = 0;
18781 #ifdef MULTIPLE_HEAPS
18782 int thread = heap_number;
18783 #endif //MULTIPLE_HEAPS
18785 exclusive_sync* loh_alloc_lock = 0;
18787 dprintf (2,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
18788 #ifdef MULTIPLE_HEAPS
18789 // We don't have each heap scan all heaps concurrently because we are worried about
18790 // multiple threads calling things like find_first_object.
18791 int h_start = (concurrent_p ? heap_number : 0);
18792 int h_end = (concurrent_p ? (heap_number + 1) : n_heaps);
18793 for (int hi = h_start; hi < h_end; hi++)
18795 gc_heap* hp = (concurrent_p ? this : g_heaps [(heap_number + hi) % n_heaps]);
18801 #endif //MULTIPLE_HEAPS
18802 BOOL small_object_segments = TRUE;
18803 int align_const = get_alignment_constant (small_object_segments);
18804 generation* gen = hp->generation_of (condemned_gen_number);
18805 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
18806 PREFIX_ASSUME(seg != NULL);
18807 loh_alloc_lock = hp->bgc_alloc_lock;
18809 uint8_t* o = hp->background_first_overflow (min_add,
18812 small_object_segments);
18816 while ((o < hp->background_seg_end (seg, concurrent_p)) && (o <= max_add))
18818 dprintf (3, ("considering %Ix", (size_t)o));
18822 if (concurrent_p && !small_object_segments)
18824 loh_alloc_lock->bgc_mark_set (o);
18826 if (((CObjectHeader*)o)->IsFree())
18828 s = unused_array_size (o);
18840 if (background_object_marked (o, FALSE) && contain_pointers_or_collectible (o))
18842 total_marked_objects++;
18843 go_through_object_cl (method_table(o), o, s, poo,
18844 uint8_t* oo = *poo;
18845 background_mark_object (oo THREAD_NUMBER_ARG);
18849 if (concurrent_p && !small_object_segments)
18851 loh_alloc_lock->bgc_mark_done ();
18854 o = o + Align (s, align_const);
18862 dprintf (2, ("went through overflow objects in segment %Ix (%d) (so far %Id marked)",
18863 heap_segment_mem (seg), (small_object_segments ? 0 : 1), total_marked_objects));
18865 if ((concurrent_p && (seg == hp->saved_overflow_ephemeral_seg)) ||
18866 (seg = heap_segment_next_in_range (seg)) == 0)
18868 if (small_object_segments)
18872 current_bgc_state = bgc_overflow_loh;
18875 dprintf (2, ("h%d: SOH: ov-mo: %Id", heap_number, total_marked_objects));
18876 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
18877 concurrent_print_time_delta (concurrent_p ? "Cov SOH" : "Nov SOH");
18878 total_marked_objects = 0;
18879 small_object_segments = FALSE;
18880 align_const = get_alignment_constant (small_object_segments);
18881 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
18883 PREFIX_ASSUME(seg != NULL);
18885 o = max (heap_segment_mem (seg), min_add);
18890 dprintf (GTC_LOG, ("h%d: LOH: ov-mo: %Id", heap_number, total_marked_objects));
18891 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
18897 o = hp->background_first_overflow (min_add,
18900 small_object_segments);
18907 BOOL gc_heap::background_process_mark_overflow (BOOL concurrent_p)
18909 BOOL grow_mark_array_p = TRUE;
18913 assert (!processed_soh_overflow_p);
18915 if ((background_max_overflow_address != 0) &&
18916 (background_min_overflow_address != MAX_PTR))
18918 // We have overflow to process but we know we can't process the ephemeral generations
18919 // now (we actually could process till the current gen1 start but since we are going to
18920 // make overflow per segment, for now I'll just stop at the saved gen1 start.
18921 saved_overflow_ephemeral_seg = ephemeral_heap_segment;
18922 background_max_soh_overflow_address = heap_segment_reserved (saved_overflow_ephemeral_seg);
18923 background_min_soh_overflow_address = generation_allocation_start (generation_of (max_generation-1));
18928 assert ((saved_overflow_ephemeral_seg == 0) ||
18929 ((background_max_soh_overflow_address != 0) &&
18930 (background_min_soh_overflow_address != MAX_PTR)));
18932 if (!processed_soh_overflow_p)
18934 // if there was no more overflow we just need to process what we didn't process
18935 // on the saved ephemeral segment.
18936 if ((background_max_overflow_address == 0) && (background_min_overflow_address == MAX_PTR))
18938 dprintf (2, ("final processing mark overflow - no more overflow since last time"));
18939 grow_mark_array_p = FALSE;
18942 background_min_overflow_address = min (background_min_overflow_address,
18943 background_min_soh_overflow_address);
18944 background_max_overflow_address = max (background_max_overflow_address,
18945 background_max_soh_overflow_address);
18946 processed_soh_overflow_p = TRUE;
18950 BOOL overflow_p = FALSE;
18952 if ((! ((background_max_overflow_address == 0)) ||
18953 ! ((background_min_overflow_address == MAX_PTR))))
18957 if (grow_mark_array_p)
18959 // Try to grow the array.
18960 size_t new_size = max (MARK_STACK_INITIAL_LENGTH, 2*background_mark_stack_array_length);
18962 if ((new_size * sizeof(mark)) > 100*1024)
18964 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
18966 new_size = min(new_max_size, new_size);
18969 if ((background_mark_stack_array_length < new_size) &&
18970 ((new_size - background_mark_stack_array_length) > (background_mark_stack_array_length / 2)))
18972 dprintf (2, ("h%d: ov grow to %Id", heap_number, new_size));
18974 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
18977 delete background_mark_stack_array;
18978 background_mark_stack_array = tmp;
18979 background_mark_stack_array_length = new_size;
18980 background_mark_stack_tos = background_mark_stack_array;
18986 grow_mark_array_p = TRUE;
18989 uint8_t* min_add = background_min_overflow_address;
18990 uint8_t* max_add = background_max_overflow_address;
18992 background_max_overflow_address = 0;
18993 background_min_overflow_address = MAX_PTR;
18995 background_process_mark_overflow_internal (max_generation, min_add, max_add, concurrent_p);
19005 #endif //BACKGROUND_GC
19008 void gc_heap::mark_through_object (uint8_t* oo, BOOL mark_class_object_p THREAD_NUMBER_DCL)
19010 #ifndef COLLECTIBLE_CLASS
19011 UNREFERENCED_PARAMETER(mark_class_object_p);
19012 BOOL to_mark_class_object = FALSE;
19013 #else //COLLECTIBLE_CLASS
19014 BOOL to_mark_class_object = (mark_class_object_p && (is_collectible(oo)));
19015 #endif //COLLECTIBLE_CLASS
19016 if (contain_pointers (oo) || to_mark_class_object)
19018 dprintf(3,( "Marking through %Ix", (size_t)oo));
19019 size_t s = size (oo);
19021 #ifdef COLLECTIBLE_CLASS
19022 if (to_mark_class_object)
19024 uint8_t* class_obj = get_class_object (oo);
19025 mark_object (class_obj THREAD_NUMBER_ARG);
19027 #endif //COLLECTIBLE_CLASS
19029 if (contain_pointers (oo))
19031 go_through_object_nostart (method_table(oo), oo, s, po,
19033 mark_object (o THREAD_NUMBER_ARG);
19039 size_t gc_heap::get_total_heap_size()
19041 size_t total_heap_size = 0;
19043 #ifdef MULTIPLE_HEAPS
19046 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19048 gc_heap* hp2 = gc_heap::g_heaps [hn];
19049 total_heap_size += hp2->generation_size (max_generation + 1) + hp2->generation_sizes (hp2->generation_of (max_generation));
19052 total_heap_size = generation_size (max_generation + 1) + generation_sizes (generation_of (max_generation));
19053 #endif //MULTIPLE_HEAPS
19055 return total_heap_size;
19058 size_t gc_heap::get_total_fragmentation()
19060 size_t total_fragmentation = 0;
19062 #ifdef MULTIPLE_HEAPS
19063 for (int i = 0; i < gc_heap::n_heaps; i++)
19065 gc_heap* hp = gc_heap::g_heaps[i];
19066 #else //MULTIPLE_HEAPS
19068 gc_heap* hp = pGenGCHeap;
19069 #endif //MULTIPLE_HEAPS
19070 for (int i = 0; i <= (max_generation + 1); i++)
19072 generation* gen = hp->generation_of (i);
19073 total_fragmentation += (generation_free_list_space (gen) + generation_free_obj_space (gen));
19077 return total_fragmentation;
19080 size_t gc_heap::committed_size()
19082 generation* gen = generation_of (max_generation);
19083 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
19084 size_t total_committed = 0;
19088 total_committed += heap_segment_committed (seg) - (uint8_t*)seg;
19090 seg = heap_segment_next (seg);
19093 if (gen != large_object_generation)
19095 gen = generation_of (max_generation + 1);
19096 seg = generation_start_segment (gen);
19103 return total_committed;
19106 size_t gc_heap::get_total_committed_size()
19108 size_t total_committed = 0;
19110 #ifdef MULTIPLE_HEAPS
19113 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19115 gc_heap* hp = gc_heap::g_heaps [hn];
19116 total_committed += hp->committed_size();
19119 total_committed = committed_size();
19120 #endif //MULTIPLE_HEAPS
19122 return total_committed;
19125 void gc_heap::get_memory_info (uint32_t* memory_load,
19126 uint64_t* available_physical,
19127 uint64_t* available_page_file)
19129 GCToOSInterface::GetMemoryStatus(memory_load, available_physical, available_page_file);
19132 void fire_mark_event (int heap_num, int root_type, size_t bytes_marked)
19134 dprintf (DT_LOG_0, ("-----------[%d]mark %d: %Id", heap_num, root_type, bytes_marked));
19135 FIRE_EVENT(GCMarkWithType, heap_num, root_type, bytes_marked);
19138 //returns TRUE is an overflow happened.
19139 BOOL gc_heap::process_mark_overflow(int condemned_gen_number)
19141 size_t last_promoted_bytes = promoted_bytes (heap_number);
19142 BOOL overflow_p = FALSE;
19144 if ((! (max_overflow_address == 0) ||
19145 ! (min_overflow_address == MAX_PTR)))
19148 // Try to grow the array.
19150 max (MARK_STACK_INITIAL_LENGTH, 2*mark_stack_array_length);
19152 if ((new_size * sizeof(mark)) > 100*1024)
19154 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
19156 new_size = min(new_max_size, new_size);
19159 if ((mark_stack_array_length < new_size) &&
19160 ((new_size - mark_stack_array_length) > (mark_stack_array_length / 2)))
19162 mark* tmp = new (nothrow) mark [new_size];
19165 delete mark_stack_array;
19166 mark_stack_array = tmp;
19167 mark_stack_array_length = new_size;
19171 uint8_t* min_add = min_overflow_address;
19172 uint8_t* max_add = max_overflow_address;
19173 max_overflow_address = 0;
19174 min_overflow_address = MAX_PTR;
19175 process_mark_overflow_internal (condemned_gen_number, min_add, max_add);
19179 size_t current_promoted_bytes = promoted_bytes (heap_number);
19181 if (current_promoted_bytes != last_promoted_bytes)
19182 fire_mark_event (heap_number, ETW::GC_ROOT_OVERFLOW, (current_promoted_bytes - last_promoted_bytes));
19186 void gc_heap::process_mark_overflow_internal (int condemned_gen_number,
19187 uint8_t* min_add, uint8_t* max_add)
19189 #ifdef MULTIPLE_HEAPS
19190 int thread = heap_number;
19191 #endif //MULTIPLE_HEAPS
19192 BOOL full_p = (condemned_gen_number == max_generation);
19194 dprintf(3,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
19195 #ifdef MULTIPLE_HEAPS
19196 for (int hi = 0; hi < n_heaps; hi++)
19198 gc_heap* hp = g_heaps [(heap_number + hi) % n_heaps];
19204 #endif //MULTIPLE_HEAPS
19205 BOOL small_object_segments = TRUE;
19206 int align_const = get_alignment_constant (small_object_segments);
19207 generation* gen = hp->generation_of (condemned_gen_number);
19208 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
19210 PREFIX_ASSUME(seg != NULL);
19211 uint8_t* o = max (heap_segment_mem (seg), min_add);
19214 uint8_t* end = heap_segment_allocated (seg);
19216 while ((o < end) && (o <= max_add))
19218 assert ((min_add <= o) && (max_add >= o));
19219 dprintf (3, ("considering %Ix", (size_t)o));
19222 mark_through_object (o, TRUE THREAD_NUMBER_ARG);
19225 o = o + Align (size (o), align_const);
19228 if (( seg = heap_segment_next_in_range (seg)) == 0)
19230 if (small_object_segments && full_p)
19232 small_object_segments = FALSE;
19233 align_const = get_alignment_constant (small_object_segments);
19234 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
19236 PREFIX_ASSUME(seg != NULL);
19238 o = max (heap_segment_mem (seg), min_add);
19248 o = max (heap_segment_mem (seg), min_add);
19255 // Scanning for promotion for dependent handles need special handling. Because the primary holds a strong
19256 // reference to the secondary (when the primary itself is reachable) and this can cause a cascading series of
19257 // promotions (the secondary of one handle is or promotes the primary of another) we might need to perform the
19258 // promotion scan multiple times.
19259 // This helper encapsulates the logic to complete all dependent handle promotions when running a server GC. It
19260 // also has the effect of processing any mark stack overflow.
19262 #ifdef MULTIPLE_HEAPS
19263 // When multiple heaps are enabled we have must utilize a more complex algorithm in order to keep all the GC
19264 // worker threads synchronized. The algorithms are sufficiently divergent that we have different
19265 // implementations based on whether MULTIPLE_HEAPS is defined or not.
19267 // Define some static variables used for synchronization in the method below. These should really be defined
19268 // locally but MSVC complains when the VOLATILE macro is expanded into an instantiation of the Volatile class.
19270 // A note about the synchronization used within this method. Communication between the worker threads is
19271 // achieved via two shared booleans (defined below). These both act as latches that are transitioned only from
19272 // false -> true by unsynchronized code. They are only read or reset to false by a single thread under the
19273 // protection of a join.
19274 static VOLATILE(BOOL) s_fUnpromotedHandles = FALSE;
19275 static VOLATILE(BOOL) s_fUnscannedPromotions = FALSE;
19276 static VOLATILE(BOOL) s_fScanRequired;
19277 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19279 // Whenever we call this method there may have been preceding object promotions. So set
19280 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19281 // based on the how the scanning proceeded).
19282 s_fUnscannedPromotions = TRUE;
19284 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
19285 // the state of this thread's portion of the dependent handle table. That's because promotions on other
19286 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
19287 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
19288 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
19289 // as all the others or they'll get out of step).
19292 // The various worker threads are all currently racing in this code. We need to work out if at least
19293 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
19294 // dependent handle table when both of the following conditions apply:
19295 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
19296 // object happens to correspond to a primary in one of our handles we might potentially have to
19297 // promote the associated secondary).
19298 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
19300 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
19301 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
19302 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
19303 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
19304 // follows below. Note that we can't read this outside of the join since on any iteration apart from
19305 // the first threads will be racing between reading this value and completing their previous
19306 // iteration's table scan.
19308 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
19309 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
19310 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
19311 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
19312 // we're safely joined.
19313 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19314 s_fUnpromotedHandles = TRUE;
19316 // Synchronize all the threads so we can read our state variables safely. The shared variable
19317 // s_fScanRequired, indicating whether we should scan the tables or terminate the loop, will be set by
19318 // a single thread inside the join.
19319 gc_t_join.join(this, gc_join_scan_dependent_handles);
19320 if (gc_t_join.joined())
19322 // We're synchronized so it's safe to read our shared state variables. We update another shared
19323 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
19324 // the loop. We scan if there has been at least one object promotion since last time and at least
19325 // one thread has a dependent handle table with a potential handle promotion possible.
19326 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
19328 // Reset our shared state variables (ready to be set again on this scan or with a good initial
19329 // value for the next call if we're terminating the loop).
19330 s_fUnscannedPromotions = FALSE;
19331 s_fUnpromotedHandles = FALSE;
19333 if (!s_fScanRequired)
19335 // We're terminating the loop. Perform any last operations that require single threaded access.
19336 if (!initial_scan_p)
19338 // On the second invocation we reconcile all mark overflow ranges across the heaps. This can help
19339 // load balance if some of the heaps have an abnormally large workload.
19340 uint8_t* all_heaps_max = 0;
19341 uint8_t* all_heaps_min = MAX_PTR;
19343 for (i = 0; i < n_heaps; i++)
19345 if (all_heaps_max < g_heaps[i]->max_overflow_address)
19346 all_heaps_max = g_heaps[i]->max_overflow_address;
19347 if (all_heaps_min > g_heaps[i]->min_overflow_address)
19348 all_heaps_min = g_heaps[i]->min_overflow_address;
19350 for (i = 0; i < n_heaps; i++)
19352 g_heaps[i]->max_overflow_address = all_heaps_max;
19353 g_heaps[i]->min_overflow_address = all_heaps_min;
19358 // Restart all the workers.
19359 dprintf(3, ("Starting all gc thread mark stack overflow processing"));
19360 gc_t_join.restart();
19363 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19364 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
19365 // global flag indicating that at least one object promotion may have occurred (the usual comment
19366 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
19367 // exit the method since we unconditionally set this variable on method entry anyway).
19368 if (process_mark_overflow(condemned_gen_number))
19369 s_fUnscannedPromotions = TRUE;
19371 // If we decided that no scan was required we can terminate the loop now.
19372 if (!s_fScanRequired)
19375 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
19376 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
19377 // could miss noting the promotion of some primary objects).
19378 gc_t_join.join(this, gc_join_rescan_dependent_handles);
19379 if (gc_t_join.joined())
19381 // Restart all the workers.
19382 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
19383 gc_t_join.restart();
19386 // If the portion of the dependent handle table managed by this worker has handles that could still be
19387 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
19388 // could require a rescan of handles on this or other workers.
19389 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19390 if (GCScan::GcDhReScan(sc))
19391 s_fUnscannedPromotions = TRUE;
19394 #else //MULTIPLE_HEAPS
19395 // Non-multiple heap version of scan_dependent_handles: much simpler without the need to keep multiple worker
19396 // threads synchronized.
19397 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19399 UNREFERENCED_PARAMETER(initial_scan_p);
19401 // Whenever we call this method there may have been preceding object promotions. So set
19402 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19403 // based on the how the scanning proceeded).
19404 bool fUnscannedPromotions = true;
19406 // Loop until there are either no more dependent handles that can have their secondary promoted or we've
19407 // managed to perform a scan without promoting anything new.
19408 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
19410 // On each iteration of the loop start with the assumption that no further objects have been promoted.
19411 fUnscannedPromotions = false;
19413 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19414 // being visible. If there was an overflow (process_mark_overflow returned true) then additional
19415 // objects now appear to be promoted and we should set the flag.
19416 if (process_mark_overflow(condemned_gen_number))
19417 fUnscannedPromotions = true;
19419 // Perform the scan and set the flag if any promotions resulted.
19420 if (GCScan::GcDhReScan(sc))
19421 fUnscannedPromotions = true;
19424 // Process any mark stack overflow that may have resulted from scanning handles (or if we didn't need to
19425 // scan any handles at all this is the processing of overflows that may have occurred prior to this method
19427 process_mark_overflow(condemned_gen_number);
19429 #endif //MULTIPLE_HEAPS
19431 void gc_heap::mark_phase (int condemned_gen_number, BOOL mark_only_p)
19433 assert (settings.concurrent == FALSE);
19436 sc.thread_number = heap_number;
19437 sc.promotion = TRUE;
19438 sc.concurrent = FALSE;
19440 dprintf(2,("---- Mark Phase condemning %d ----", condemned_gen_number));
19441 BOOL full_p = (condemned_gen_number == max_generation);
19446 start = GetCycleCount32();
19449 int gen_to_init = condemned_gen_number;
19450 if (condemned_gen_number == max_generation)
19452 gen_to_init = max_generation + 1;
19454 for (int gen_idx = 0; gen_idx <= gen_to_init; gen_idx++)
19456 dynamic_data* dd = dynamic_data_of (gen_idx);
19457 dd_begin_data_size (dd) = generation_size (gen_idx) -
19458 dd_fragmentation (dd) -
19459 Align (size (generation_allocation_start (generation_of (gen_idx))));
19460 dprintf (2, ("begin data size for gen%d is %Id", gen_idx, dd_begin_data_size (dd)));
19461 dd_survived_size (dd) = 0;
19462 dd_pinned_survived_size (dd) = 0;
19463 dd_artificial_pinned_survived_size (dd) = 0;
19464 dd_added_pinned_size (dd) = 0;
19466 dd_padding_size (dd) = 0;
19467 #endif //SHORT_PLUGS
19468 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
19469 dd_num_npinned_plugs (dd) = 0;
19470 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
19473 #ifdef FFIND_OBJECT
19474 if (gen0_must_clear_bricks > 0)
19475 gen0_must_clear_bricks--;
19476 #endif //FFIND_OBJECT
19478 size_t last_promoted_bytes = 0;
19480 promoted_bytes (heap_number) = 0;
19481 reset_mark_stack();
19484 memset (&snoop_stat, 0, sizeof(snoop_stat));
19485 snoop_stat.heap_index = heap_number;
19486 #endif //SNOOP_STATS
19491 //initialize the mark stack
19492 for (int i = 0; i < max_snoop_level; i++)
19494 ((uint8_t**)(mark_stack_array))[i] = 0;
19497 mark_stack_busy() = 1;
19499 #endif //MH_SC_MARK
19501 static uint32_t num_sizedrefs = 0;
19504 static BOOL do_mark_steal_p = FALSE;
19505 #endif //MH_SC_MARK
19507 #ifdef MULTIPLE_HEAPS
19508 gc_t_join.join(this, gc_join_begin_mark_phase);
19509 if (gc_t_join.joined())
19511 #endif //MULTIPLE_HEAPS
19513 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
19515 #ifdef MULTIPLE_HEAPS
19520 size_t total_heap_size = get_total_heap_size();
19522 if (total_heap_size > (100 * 1024 * 1024))
19524 do_mark_steal_p = TRUE;
19528 do_mark_steal_p = FALSE;
19533 do_mark_steal_p = FALSE;
19535 #endif //MH_SC_MARK
19537 gc_t_join.restart();
19539 #endif //MULTIPLE_HEAPS
19544 //set up the mark lists from g_mark_list
19545 assert (g_mark_list);
19546 #ifdef MULTIPLE_HEAPS
19547 mark_list = &g_mark_list [heap_number*mark_list_size];
19549 mark_list = g_mark_list;
19550 #endif //MULTIPLE_HEAPS
19551 //dont use the mark list for full gc
19552 //because multiple segments are more complex to handle and the list
19553 //is likely to overflow
19554 if (condemned_gen_number != max_generation)
19555 mark_list_end = &mark_list [mark_list_size-1];
19557 mark_list_end = &mark_list [0];
19558 mark_list_index = &mark_list [0];
19561 shigh = (uint8_t*) 0;
19564 //%type% category = quote (mark);
19566 if ((condemned_gen_number == max_generation) && (num_sizedrefs > 0))
19568 GCScan::GcScanSizedRefs(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
19569 fire_mark_event (heap_number, ETW::GC_ROOT_SIZEDREF, (promoted_bytes (heap_number) - last_promoted_bytes));
19570 last_promoted_bytes = promoted_bytes (heap_number);
19572 #ifdef MULTIPLE_HEAPS
19573 gc_t_join.join(this, gc_join_scan_sizedref_done);
19574 if (gc_t_join.joined())
19576 dprintf(3, ("Done with marking all sized refs. Starting all gc thread for marking other strong roots"));
19577 gc_t_join.restart();
19579 #endif //MULTIPLE_HEAPS
19582 dprintf(3,("Marking Roots"));
19584 GCScan::GcScanRoots(GCHeap::Promote,
19585 condemned_gen_number, max_generation,
19588 fire_mark_event (heap_number, ETW::GC_ROOT_STACK, (promoted_bytes (heap_number) - last_promoted_bytes));
19589 last_promoted_bytes = promoted_bytes (heap_number);
19591 #ifdef BACKGROUND_GC
19592 if (recursive_gc_sync::background_running_p())
19594 scan_background_roots (GCHeap::Promote, heap_number, &sc);
19596 #endif //BACKGROUND_GC
19598 #ifdef FEATURE_PREMORTEM_FINALIZATION
19599 dprintf(3, ("Marking finalization data"));
19600 finalize_queue->GcScanRoots(GCHeap::Promote, heap_number, 0);
19601 #endif // FEATURE_PREMORTEM_FINALIZATION
19603 fire_mark_event (heap_number, ETW::GC_ROOT_FQ, (promoted_bytes (heap_number) - last_promoted_bytes));
19604 last_promoted_bytes = promoted_bytes (heap_number);
19609 dprintf(3,("Marking handle table"));
19610 GCScan::GcScanHandles(GCHeap::Promote,
19611 condemned_gen_number, max_generation,
19613 fire_mark_event (heap_number, ETW::GC_ROOT_HANDLES, (promoted_bytes (heap_number) - last_promoted_bytes));
19614 last_promoted_bytes = promoted_bytes (heap_number);
19618 size_t promoted_before_cards = promoted_bytes (heap_number);
19621 dprintf (3, ("before cards: %Id", promoted_before_cards));
19625 #ifdef MULTIPLE_HEAPS
19626 if (gc_t_join.r_join(this, gc_r_join_update_card_bundle))
19628 #endif //MULTIPLE_HEAPS
19630 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
19631 // If we are manually managing card bundles, every write to the card table should already be
19632 // accounted for in the card bundle table so there's nothing to update here.
19633 update_card_table_bundle();
19635 if (card_bundles_enabled())
19637 verify_card_bundles();
19640 #ifdef MULTIPLE_HEAPS
19641 gc_t_join.r_restart();
19643 #endif //MULTIPLE_HEAPS
19644 #endif //CARD_BUNDLE
19646 card_fn mark_object_fn = &gc_heap::mark_object_simple;
19647 #ifdef HEAP_ANALYZE
19648 heap_analyze_success = TRUE;
19649 if (heap_analyze_enabled)
19651 internal_root_array_index = 0;
19653 current_obj_size = 0;
19654 mark_object_fn = &gc_heap::ha_mark_object_simple;
19656 #endif //HEAP_ANALYZE
19658 dprintf(3,("Marking cross generation pointers"));
19659 mark_through_cards_for_segments (mark_object_fn, FALSE);
19661 dprintf(3,("Marking cross generation pointers for large objects"));
19662 mark_through_cards_for_large_objects (mark_object_fn, FALSE);
19664 dprintf (3, ("marked by cards: %Id",
19665 (promoted_bytes (heap_number) - promoted_before_cards)));
19666 fire_mark_event (heap_number, ETW::GC_ROOT_OLDER, (promoted_bytes (heap_number) - last_promoted_bytes));
19667 last_promoted_bytes = promoted_bytes (heap_number);
19672 if (do_mark_steal_p)
19676 #endif //MH_SC_MARK
19678 // Dependent handles need to be scanned with a special algorithm (see the header comment on
19679 // scan_dependent_handles for more detail). We perform an initial scan without synchronizing with other
19680 // worker threads or processing any mark stack overflow. This is not guaranteed to complete the operation
19681 // but in a common case (where there are no dependent handles that are due to be collected) it allows us
19682 // to optimize away further scans. The call to scan_dependent_handles is what will cycle through more
19683 // iterations if required and will also perform processing of any mark stack overflow once the dependent
19684 // handle table has been fully promoted.
19685 GCScan::GcDhInitialScan(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
19686 scan_dependent_handles(condemned_gen_number, &sc, true);
19688 #ifdef MULTIPLE_HEAPS
19689 dprintf(3, ("Joining for short weak handle scan"));
19690 gc_t_join.join(this, gc_join_null_dead_short_weak);
19691 if (gc_t_join.joined())
19692 #endif //MULTIPLE_HEAPS
19694 #ifdef HEAP_ANALYZE
19695 heap_analyze_enabled = FALSE;
19696 GCToEEInterface::AnalyzeSurvivorsFinished(condemned_gen_number);
19697 #endif // HEAP_ANALYZE
19698 GCToEEInterface::AfterGcScanRoots (condemned_gen_number, max_generation, &sc);
19700 #ifdef MULTIPLE_HEAPS
19703 // we used r_join and need to reinitialize states for it here.
19704 gc_t_join.r_init();
19707 //start all threads on the roots.
19708 dprintf(3, ("Starting all gc thread for short weak handle scan"));
19709 gc_t_join.restart();
19710 #endif //MULTIPLE_HEAPS
19714 // null out the target of short weakref that were not promoted.
19715 GCScan::GcShortWeakPtrScan(GCHeap::Promote, condemned_gen_number, max_generation,&sc);
19717 // MTHTS: keep by single thread
19718 #ifdef MULTIPLE_HEAPS
19719 dprintf(3, ("Joining for finalization"));
19720 gc_t_join.join(this, gc_join_scan_finalization);
19721 if (gc_t_join.joined())
19722 #endif //MULTIPLE_HEAPS
19725 #ifdef MULTIPLE_HEAPS
19726 //start all threads on the roots.
19727 dprintf(3, ("Starting all gc thread for Finalization"));
19728 gc_t_join.restart();
19729 #endif //MULTIPLE_HEAPS
19732 //Handle finalization.
19733 size_t promoted_bytes_live = promoted_bytes (heap_number);
19735 #ifdef FEATURE_PREMORTEM_FINALIZATION
19736 dprintf (3, ("Finalize marking"));
19737 finalize_queue->ScanForFinalization (GCHeap::Promote, condemned_gen_number, mark_only_p, __this);
19739 GCToEEInterface::DiagWalkFReachableObjects(__this);
19740 #endif // FEATURE_PREMORTEM_FINALIZATION
19742 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
19743 // for finalization. As before scan_dependent_handles will also process any mark stack overflow.
19744 scan_dependent_handles(condemned_gen_number, &sc, false);
19746 #ifdef MULTIPLE_HEAPS
19747 dprintf(3, ("Joining for weak pointer deletion"));
19748 gc_t_join.join(this, gc_join_null_dead_long_weak);
19749 if (gc_t_join.joined())
19751 //start all threads on the roots.
19752 dprintf(3, ("Starting all gc thread for weak pointer deletion"));
19753 gc_t_join.restart();
19755 #endif //MULTIPLE_HEAPS
19757 // null out the target of long weakref that were not promoted.
19758 GCScan::GcWeakPtrScan (GCHeap::Promote, condemned_gen_number, max_generation, &sc);
19760 // MTHTS: keep by single thread
19761 #ifdef MULTIPLE_HEAPS
19763 #ifdef PARALLEL_MARK_LIST_SORT
19764 // unsigned long start = GetCycleCount32();
19766 // printf("sort_mark_list took %u cycles\n", GetCycleCount32() - start);
19767 #endif //PARALLEL_MARK_LIST_SORT
19770 dprintf (3, ("Joining for sync block cache entry scanning"));
19771 gc_t_join.join(this, gc_join_null_dead_syncblk);
19772 if (gc_t_join.joined())
19773 #endif //MULTIPLE_HEAPS
19775 // scan for deleted entries in the syncblk cache
19776 GCScan::GcWeakPtrScanBySingleThread (condemned_gen_number, max_generation, &sc);
19778 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
19781 size_t promoted_all_heaps = 0;
19782 #ifdef MULTIPLE_HEAPS
19783 for (int i = 0; i < n_heaps; i++)
19785 promoted_all_heaps += promoted_bytes (i);
19788 promoted_all_heaps = promoted_bytes (heap_number);
19789 #endif //MULTIPLE_HEAPS
19790 SystemDomain::RecordTotalSurvivedBytes (promoted_all_heaps);
19792 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
19794 #ifdef MULTIPLE_HEAPS
19797 #ifndef PARALLEL_MARK_LIST_SORT
19798 //compact g_mark_list and sort it.
19799 combine_mark_lists();
19800 #endif //PARALLEL_MARK_LIST_SORT
19803 //decide on promotion
19804 if (!settings.promotion)
19807 for (int n = 0; n <= condemned_gen_number;n++)
19809 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.1);
19812 for (int i = 0; i < n_heaps; i++)
19814 dynamic_data* dd = g_heaps[i]->dynamic_data_of (min (condemned_gen_number +1,
19816 size_t older_gen_size = (dd_current_size (dd) +
19817 (dd_desired_allocation (dd) -
19818 dd_new_allocation (dd)));
19820 if ((m > (older_gen_size)) ||
19821 (promoted_bytes (i) > m))
19823 settings.promotion = TRUE;
19829 if (do_mark_steal_p)
19831 size_t objects_checked_count = 0;
19832 size_t zero_ref_count = 0;
19833 size_t objects_marked_count = 0;
19834 size_t check_level_count = 0;
19835 size_t busy_count = 0;
19836 size_t interlocked_count = 0;
19837 size_t partial_mark_parent_count = 0;
19838 size_t stolen_or_pm_count = 0;
19839 size_t stolen_entry_count = 0;
19840 size_t pm_not_ready_count = 0;
19841 size_t normal_count = 0;
19842 size_t stack_bottom_clear_count = 0;
19844 for (int i = 0; i < n_heaps; i++)
19846 gc_heap* hp = g_heaps[i];
19847 hp->print_snoop_stat();
19848 objects_checked_count += hp->snoop_stat.objects_checked_count;
19849 zero_ref_count += hp->snoop_stat.zero_ref_count;
19850 objects_marked_count += hp->snoop_stat.objects_marked_count;
19851 check_level_count += hp->snoop_stat.check_level_count;
19852 busy_count += hp->snoop_stat.busy_count;
19853 interlocked_count += hp->snoop_stat.interlocked_count;
19854 partial_mark_parent_count += hp->snoop_stat.partial_mark_parent_count;
19855 stolen_or_pm_count += hp->snoop_stat.stolen_or_pm_count;
19856 stolen_entry_count += hp->snoop_stat.stolen_entry_count;
19857 pm_not_ready_count += hp->snoop_stat.pm_not_ready_count;
19858 normal_count += hp->snoop_stat.normal_count;
19859 stack_bottom_clear_count += hp->snoop_stat.stack_bottom_clear_count;
19864 printf ("-------total stats-------\n");
19865 printf ("%8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
19866 "checked", "zero", "marked", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
19867 printf ("%8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
19868 objects_checked_count,
19870 objects_marked_count,
19874 partial_mark_parent_count,
19875 stolen_or_pm_count,
19876 stolen_entry_count,
19877 pm_not_ready_count,
19879 stack_bottom_clear_count);
19881 #endif //SNOOP_STATS
19883 //start all threads.
19884 dprintf(3, ("Starting all threads for end of mark phase"));
19885 gc_t_join.restart();
19886 #else //MULTIPLE_HEAPS
19888 //decide on promotion
19889 if (!settings.promotion)
19892 for (int n = 0; n <= condemned_gen_number;n++)
19894 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.06);
19896 dynamic_data* dd = dynamic_data_of (min (condemned_gen_number +1,
19898 size_t older_gen_size = (dd_current_size (dd) +
19899 (dd_desired_allocation (dd) -
19900 dd_new_allocation (dd)));
19902 dprintf (2, ("promotion threshold: %Id, promoted bytes: %Id size n+1: %Id",
19903 m, promoted_bytes (heap_number), older_gen_size));
19905 if ((m > older_gen_size) ||
19906 (promoted_bytes (heap_number) > m))
19908 settings.promotion = TRUE;
19912 #endif //MULTIPLE_HEAPS
19915 #ifdef MULTIPLE_HEAPS
19917 #ifdef PARALLEL_MARK_LIST_SORT
19918 // start = GetCycleCount32();
19919 merge_mark_lists();
19920 // printf("merge_mark_lists took %u cycles\n", GetCycleCount32() - start);
19921 #endif //PARALLEL_MARK_LIST_SORT
19923 #endif //MULTIPLE_HEAPS
19925 #ifdef BACKGROUND_GC
19926 total_promoted_bytes = promoted_bytes (heap_number);
19927 #endif //BACKGROUND_GC
19929 promoted_bytes (heap_number) -= promoted_bytes_live;
19932 finish = GetCycleCount32();
19933 mark_time = finish - start;
19936 dprintf(2,("---- End of mark phase ----"));
19940 void gc_heap::pin_object (uint8_t* o, uint8_t** ppObject, uint8_t* low, uint8_t* high)
19942 dprintf (3, ("Pinning %Ix", (size_t)o));
19943 if ((o >= low) && (o < high))
19945 dprintf(3,("^%Ix^", (size_t)o));
19948 #ifdef FEATURE_EVENT_TRACE
19949 if(EVENT_ENABLED(PinObjectAtGCTime))
19951 fire_etw_pin_object_event(o, ppObject);
19953 #endif // FEATURE_EVENT_TRACE
19955 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
19956 num_pinned_objects++;
19957 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
19961 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
19962 size_t gc_heap::get_total_pinned_objects()
19964 #ifdef MULTIPLE_HEAPS
19965 size_t total_num_pinned_objects = 0;
19966 for (int i = 0; i < gc_heap::n_heaps; i++)
19968 gc_heap* hp = gc_heap::g_heaps[i];
19969 total_num_pinned_objects += hp->num_pinned_objects;
19971 return total_num_pinned_objects;
19972 #else //MULTIPLE_HEAPS
19973 return num_pinned_objects;
19974 #endif //MULTIPLE_HEAPS
19976 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
19978 void gc_heap::reset_mark_stack ()
19980 reset_pinned_queue();
19981 max_overflow_address = 0;
19982 min_overflow_address = MAX_PTR;
19985 #ifdef FEATURE_STRUCTALIGN
19987 // The word with left child, right child, and align info is laid out as follows:
19989 // | upper short word | lower short word |
19990 // |<------------> <----->|<------------> <----->|
19991 // | left child info hi| right child info lo|
19992 // x86: | 10 bits 6 bits| 10 bits 6 bits|
19994 // where left/right child are signed values and concat(info hi, info lo) is unsigned.
19996 // The "align info" encodes two numbers: the required alignment (a power of two)
19997 // and the misalignment (the number of machine words the destination address needs
19998 // to be adjusted by to provide alignment - so this number is always smaller than
19999 // the required alignment). Thus, the two can be represented as the "logical or"
20000 // of the two numbers. Note that the actual pad is computed from the misalignment
20001 // by adding the alignment iff the misalignment is non-zero and less than min_obj_size.
20004 // The number of bits in a brick.
20005 #if defined (_TARGET_AMD64_)
20006 #define brick_bits (12)
20008 #define brick_bits (11)
20009 #endif //_TARGET_AMD64_
20010 C_ASSERT(brick_size == (1 << brick_bits));
20012 // The number of bits needed to represent the offset to a child node.
20013 // "brick_bits + 1" allows us to represent a signed offset within a brick.
20014 #define child_bits (brick_bits + 1 - LOG2_PTRSIZE)
20016 // The number of bits in each of the pad hi, pad lo fields.
20017 #define pad_bits (sizeof(short) * 8 - child_bits)
20019 #define child_from_short(w) (((signed short)(w) / (1 << (pad_bits - LOG2_PTRSIZE))) & ~((1 << LOG2_PTRSIZE) - 1))
20020 #define pad_mask ((1 << pad_bits) - 1)
20021 #define pad_from_short(w) ((size_t)(w) & pad_mask)
20022 #else // FEATURE_STRUCTALIGN
20023 #define child_from_short(w) (w)
20024 #endif // FEATURE_STRUCTALIGN
20027 short node_left_child(uint8_t* node)
20029 return child_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20033 void set_node_left_child(uint8_t* node, ptrdiff_t val)
20035 assert (val > -(ptrdiff_t)brick_size);
20036 assert (val < (ptrdiff_t)brick_size);
20037 assert (Aligned (val));
20038 #ifdef FEATURE_STRUCTALIGN
20039 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20040 ((plug_and_pair*)node)[-1].m_pair.left = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20041 #else // FEATURE_STRUCTALIGN
20042 ((plug_and_pair*)node)[-1].m_pair.left = (short)val;
20043 #endif // FEATURE_STRUCTALIGN
20044 assert (node_left_child (node) == val);
20048 short node_right_child(uint8_t* node)
20050 return child_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20054 void set_node_right_child(uint8_t* node, ptrdiff_t val)
20056 assert (val > -(ptrdiff_t)brick_size);
20057 assert (val < (ptrdiff_t)brick_size);
20058 assert (Aligned (val));
20059 #ifdef FEATURE_STRUCTALIGN
20060 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20061 ((plug_and_pair*)node)[-1].m_pair.right = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20062 #else // FEATURE_STRUCTALIGN
20063 ((plug_and_pair*)node)[-1].m_pair.right = (short)val;
20064 #endif // FEATURE_STRUCTALIGN
20065 assert (node_right_child (node) == val);
20068 #ifdef FEATURE_STRUCTALIGN
20069 void node_aligninfo (uint8_t* node, int& requiredAlignment, ptrdiff_t& pad)
20071 // Extract the single-number aligninfo from the fields.
20072 short left = ((plug_and_pair*)node)[-1].m_pair.left;
20073 short right = ((plug_and_pair*)node)[-1].m_pair.right;
20074 ptrdiff_t pad_shifted = (pad_from_short(left) << pad_bits) | pad_from_short(right);
20075 ptrdiff_t aligninfo = pad_shifted * DATA_ALIGNMENT;
20077 // Replicate the topmost bit into all lower bits.
20078 ptrdiff_t x = aligninfo;
20084 // Clear all bits but the highest.
20085 requiredAlignment = (int)(x ^ (x >> 1));
20086 pad = aligninfo - requiredAlignment;
20087 pad += AdjustmentForMinPadSize(pad, requiredAlignment);
20091 ptrdiff_t node_alignpad (uint8_t* node)
20093 int requiredAlignment;
20094 ptrdiff_t alignpad;
20095 node_aligninfo (node, requiredAlignment, alignpad);
20099 void clear_node_aligninfo (uint8_t* node)
20101 ((plug_and_pair*)node)[-1].m_pair.left &= ~0 << pad_bits;
20102 ((plug_and_pair*)node)[-1].m_pair.right &= ~0 << pad_bits;
20105 void set_node_aligninfo (uint8_t* node, int requiredAlignment, ptrdiff_t pad)
20107 // Encode the alignment requirement and alignment offset as a single number
20108 // as described above.
20109 ptrdiff_t aligninfo = (size_t)requiredAlignment + (pad & (requiredAlignment-1));
20110 assert (Aligned (aligninfo));
20111 ptrdiff_t aligninfo_shifted = aligninfo / DATA_ALIGNMENT;
20112 assert (aligninfo_shifted < (1 << (pad_bits + pad_bits)));
20114 ptrdiff_t hi = aligninfo_shifted >> pad_bits;
20115 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.left) == 0);
20116 ((plug_and_pair*)node)[-1].m_pair.left |= hi;
20118 ptrdiff_t lo = aligninfo_shifted & pad_mask;
20119 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.right) == 0);
20120 ((plug_and_pair*)node)[-1].m_pair.right |= lo;
20123 int requiredAlignment2;
20125 node_aligninfo (node, requiredAlignment2, pad2);
20126 assert (requiredAlignment == requiredAlignment2);
20127 assert (pad == pad2);
20130 #endif // FEATURE_STRUCTALIGN
20133 void loh_set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20135 ptrdiff_t* place = &(((loh_obj_and_pad*)node)[-1].reloc);
20140 ptrdiff_t loh_node_relocation_distance(uint8_t* node)
20142 return (((loh_obj_and_pad*)node)[-1].reloc);
20146 ptrdiff_t node_relocation_distance (uint8_t* node)
20148 return (((plug_and_reloc*)(node))[-1].reloc & ~3);
20152 void set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20154 assert (val == (val & ~3));
20155 ptrdiff_t* place = &(((plug_and_reloc*)node)[-1].reloc);
20156 //clear the left bit and the relocation field
20162 #define node_left_p(node) (((plug_and_reloc*)(node))[-1].reloc & 2)
20164 #define set_node_left(node) ((plug_and_reloc*)(node))[-1].reloc |= 2;
20166 #ifndef FEATURE_STRUCTALIGN
20167 void set_node_realigned(uint8_t* node)
20169 ((plug_and_reloc*)(node))[-1].reloc |= 1;
20172 void clear_node_realigned(uint8_t* node)
20174 #ifdef RESPECT_LARGE_ALIGNMENT
20175 ((plug_and_reloc*)(node))[-1].reloc &= ~1;
20177 UNREFERENCED_PARAMETER(node);
20178 #endif //RESPECT_LARGE_ALIGNMENT
20180 #endif // FEATURE_STRUCTALIGN
20183 size_t node_gap_size (uint8_t* node)
20185 return ((plug_and_gap *)node)[-1].gap;
20188 void set_gap_size (uint8_t* node, size_t size)
20190 assert (Aligned (size));
20192 // clear the 2 uint32_t used by the node.
20193 ((plug_and_gap *)node)[-1].reloc = 0;
20194 ((plug_and_gap *)node)[-1].lr =0;
20195 ((plug_and_gap *)node)[-1].gap = size;
20197 assert ((size == 0 )||(size >= sizeof(plug_and_reloc)));
20201 uint8_t* gc_heap::insert_node (uint8_t* new_node, size_t sequence_number,
20202 uint8_t* tree, uint8_t* last_node)
20204 dprintf (3, ("IN: %Ix(%Ix), T: %Ix(%Ix), L: %Ix(%Ix) [%Ix]",
20205 (size_t)new_node, brick_of(new_node),
20206 (size_t)tree, brick_of(tree),
20207 (size_t)last_node, brick_of(last_node),
20209 if (power_of_two_p (sequence_number))
20211 set_node_left_child (new_node, (tree - new_node));
20212 dprintf (3, ("NT: %Ix, LC->%Ix", (size_t)new_node, (tree - new_node)));
20217 if (oddp (sequence_number))
20219 set_node_right_child (last_node, (new_node - last_node));
20220 dprintf (3, ("%Ix RC->%Ix", last_node, (new_node - last_node)));
20224 uint8_t* earlier_node = tree;
20225 size_t imax = logcount(sequence_number) - 2;
20226 for (size_t i = 0; i != imax; i++)
20228 earlier_node = earlier_node + node_right_child (earlier_node);
20230 int tmp_offset = node_right_child (earlier_node);
20231 assert (tmp_offset); // should never be empty
20232 set_node_left_child (new_node, ((earlier_node + tmp_offset ) - new_node));
20233 set_node_right_child (earlier_node, (new_node - earlier_node));
20235 dprintf (3, ("%Ix LC->%Ix, %Ix RC->%Ix",
20236 new_node, ((earlier_node + tmp_offset ) - new_node),
20237 earlier_node, (new_node - earlier_node)));
20243 size_t gc_heap::update_brick_table (uint8_t* tree, size_t current_brick,
20244 uint8_t* x, uint8_t* plug_end)
20246 dprintf (3, ("tree: %Ix, current b: %Ix, x: %Ix, plug_end: %Ix",
20247 tree, current_brick, x, plug_end));
20251 dprintf (3, ("b- %Ix->%Ix pointing to tree %Ix",
20252 current_brick, (size_t)(tree - brick_address (current_brick)), tree));
20253 set_brick (current_brick, (tree - brick_address (current_brick)));
20257 dprintf (3, ("b- %Ix->-1", current_brick));
20258 set_brick (current_brick, -1);
20260 size_t b = 1 + current_brick;
20261 ptrdiff_t offset = 0;
20262 size_t last_br = brick_of (plug_end-1);
20263 current_brick = brick_of (x-1);
20264 dprintf (3, ("ubt: %Ix->%Ix]->%Ix]", b, last_br, current_brick));
20265 while (b <= current_brick)
20269 set_brick (b, --offset);
20277 return brick_of (x);
20280 void gc_heap::plan_generation_start (generation* gen, generation* consing_gen, uint8_t* next_plug_to_allocate)
20283 // We should never demote big plugs to gen0.
20284 if (gen == youngest_generation)
20286 heap_segment* seg = ephemeral_heap_segment;
20287 size_t mark_stack_large_bos = mark_stack_bos;
20288 size_t large_plug_pos = 0;
20289 while (mark_stack_large_bos < mark_stack_tos)
20291 if (mark_stack_array[mark_stack_large_bos].len > demotion_plug_len_th)
20293 while (mark_stack_bos <= mark_stack_large_bos)
20295 size_t entry = deque_pinned_plug();
20296 size_t len = pinned_len (pinned_plug_of (entry));
20297 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20298 if (len > demotion_plug_len_th)
20300 dprintf (2, ("ps(%d): S %Ix (%Id)(%Ix)", gen->gen_num, plug, len, (plug+len)));
20302 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (consing_gen);
20303 assert(mark_stack_array[entry].len == 0 ||
20304 mark_stack_array[entry].len >= Align(min_obj_size));
20305 generation_allocation_pointer (consing_gen) = plug + len;
20306 generation_allocation_limit (consing_gen) = heap_segment_plan_allocated (seg);
20307 set_allocator_next_pin (consing_gen);
20311 mark_stack_large_bos++;
20316 generation_plan_allocation_start (gen) =
20317 allocate_in_condemned_generations (consing_gen, Align (min_obj_size), -1);
20318 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20319 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20320 if (next_plug_to_allocate)
20322 size_t dist_to_next_plug = (size_t)(next_plug_to_allocate - generation_allocation_pointer (consing_gen));
20323 if (allocation_left > dist_to_next_plug)
20325 allocation_left = dist_to_next_plug;
20328 if (allocation_left < Align (min_obj_size))
20330 generation_plan_allocation_start_size (gen) += allocation_left;
20331 generation_allocation_pointer (consing_gen) += allocation_left;
20334 dprintf (1, ("plan alloc gen%d(%Ix) start at %Ix (ptr: %Ix, limit: %Ix, next: %Ix)", gen->gen_num,
20335 generation_plan_allocation_start (gen),
20336 generation_plan_allocation_start_size (gen),
20337 generation_allocation_pointer (consing_gen), generation_allocation_limit (consing_gen),
20338 next_plug_to_allocate));
20341 void gc_heap::realloc_plan_generation_start (generation* gen, generation* consing_gen)
20343 BOOL adjacentp = FALSE;
20345 generation_plan_allocation_start (gen) =
20346 allocate_in_expanded_heap (consing_gen, Align(min_obj_size), adjacentp, 0,
20349 #endif //SHORT_PLUGS
20350 FALSE, -1 REQD_ALIGN_AND_OFFSET_ARG);
20352 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20353 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20354 if ((allocation_left < Align (min_obj_size)) &&
20355 (generation_allocation_limit (consing_gen)!=heap_segment_plan_allocated (generation_allocation_segment (consing_gen))))
20357 generation_plan_allocation_start_size (gen) += allocation_left;
20358 generation_allocation_pointer (consing_gen) += allocation_left;
20361 dprintf (1, ("plan re-alloc gen%d start at %Ix (ptr: %Ix, limit: %Ix)", gen->gen_num,
20362 generation_plan_allocation_start (consing_gen),
20363 generation_allocation_pointer (consing_gen),
20364 generation_allocation_limit (consing_gen)));
20367 void gc_heap::plan_generation_starts (generation*& consing_gen)
20369 //make sure that every generation has a planned allocation start
20370 int gen_number = settings.condemned_generation;
20371 while (gen_number >= 0)
20373 if (gen_number < max_generation)
20375 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
20377 generation* gen = generation_of (gen_number);
20378 if (0 == generation_plan_allocation_start (gen))
20380 plan_generation_start (gen, consing_gen, 0);
20381 assert (generation_plan_allocation_start (gen));
20385 // now we know the planned allocation size
20386 heap_segment_plan_allocated (ephemeral_heap_segment) =
20387 generation_allocation_pointer (consing_gen);
20390 void gc_heap::advance_pins_for_demotion (generation* gen)
20392 uint8_t* original_youngest_start = generation_allocation_start (youngest_generation);
20393 heap_segment* seg = ephemeral_heap_segment;
20395 if ((!(pinned_plug_que_empty_p())))
20397 size_t gen1_pinned_promoted = generation_pinned_allocation_compact_size (generation_of (max_generation));
20398 size_t gen1_pins_left = dd_pinned_survived_size (dynamic_data_of (max_generation - 1)) - gen1_pinned_promoted;
20399 size_t total_space_to_skip = last_gen1_pin_end - generation_allocation_pointer (gen);
20400 float pin_frag_ratio = (float)gen1_pins_left / (float)total_space_to_skip;
20401 float pin_surv_ratio = (float)gen1_pins_left / (float)(dd_survived_size (dynamic_data_of (max_generation - 1)));
20402 if ((pin_frag_ratio > 0.15) && (pin_surv_ratio > 0.30))
20404 while (!pinned_plug_que_empty_p() &&
20405 (pinned_plug (oldest_pin()) < original_youngest_start))
20407 size_t entry = deque_pinned_plug();
20408 size_t len = pinned_len (pinned_plug_of (entry));
20409 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20410 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (gen);
20411 assert(mark_stack_array[entry].len == 0 ||
20412 mark_stack_array[entry].len >= Align(min_obj_size));
20413 generation_allocation_pointer (gen) = plug + len;
20414 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20415 set_allocator_next_pin (gen);
20417 //Add the size of the pinned plug to the right pinned allocations
20418 //find out which gen this pinned plug came from
20419 int frgn = object_gennum (plug);
20420 if ((frgn != (int)max_generation) && settings.promotion)
20422 int togn = object_gennum_plan (plug);
20423 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
20426 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
20430 dprintf (2, ("skipping gap %d, pin %Ix (%Id)",
20431 pinned_len (pinned_plug_of (entry)), plug, len));
20434 dprintf (2, ("ad_p_d: PL: %Id, SL: %Id, pfr: %d, psr: %d",
20435 gen1_pins_left, total_space_to_skip, (int)(pin_frag_ratio*100), (int)(pin_surv_ratio*100)));
20439 void gc_heap::process_ephemeral_boundaries (uint8_t* x,
20440 int& active_new_gen_number,
20441 int& active_old_gen_number,
20442 generation*& consing_gen,
20443 BOOL& allocate_in_condemned)
20446 if ((active_old_gen_number > 0) &&
20447 (x >= generation_allocation_start (generation_of (active_old_gen_number - 1))))
20449 dprintf (1, ("crossing gen%d, x is %Ix", active_old_gen_number - 1, x));
20451 if (!pinned_plug_que_empty_p())
20453 dprintf (1, ("oldest pin: %Ix(%Id)",
20454 pinned_plug (oldest_pin()),
20455 (x - pinned_plug (oldest_pin()))));
20458 if (active_old_gen_number <= (settings.promotion ? (max_generation - 1) : max_generation))
20460 active_new_gen_number--;
20463 active_old_gen_number--;
20464 assert ((!settings.promotion) || (active_new_gen_number>0));
20466 if (active_new_gen_number == (max_generation - 1))
20468 #ifdef FREE_USAGE_STATS
20469 if (settings.condemned_generation == max_generation)
20471 // We need to do this before we skip the rest of the pinned plugs.
20472 generation* gen_2 = generation_of (max_generation);
20473 generation* gen_1 = generation_of (max_generation - 1);
20475 size_t total_num_pinned_free_spaces_left = 0;
20477 // We are about to allocate gen1, check to see how efficient fitting in gen2 pinned free spaces is.
20478 for (int j = 0; j < NUM_GEN_POWER2; j++)
20480 dprintf (1, ("[h%d][#%Id]2^%d: current: %Id, S: 2: %Id, 1: %Id(%Id)",
20484 gen_2->gen_current_pinned_free_spaces[j],
20485 gen_2->gen_plugs[j], gen_1->gen_plugs[j],
20486 (gen_2->gen_plugs[j] + gen_1->gen_plugs[j])));
20488 total_num_pinned_free_spaces_left += gen_2->gen_current_pinned_free_spaces[j];
20491 float pinned_free_list_efficiency = 0;
20492 size_t total_pinned_free_space = generation_allocated_in_pinned_free (gen_2) + generation_pinned_free_obj_space (gen_2);
20493 if (total_pinned_free_space != 0)
20495 pinned_free_list_efficiency = (float)(generation_allocated_in_pinned_free (gen_2)) / (float)total_pinned_free_space;
20498 dprintf (1, ("[h%d] gen2 allocated %Id bytes with %Id bytes pinned free spaces (effi: %d%%), %Id (%Id) left",
20500 generation_allocated_in_pinned_free (gen_2),
20501 total_pinned_free_space,
20502 (int)(pinned_free_list_efficiency * 100),
20503 generation_pinned_free_obj_space (gen_2),
20504 total_num_pinned_free_spaces_left));
20506 #endif //FREE_USAGE_STATS
20508 //Go past all of the pinned plugs for this generation.
20509 while (!pinned_plug_que_empty_p() &&
20510 (!in_range_for_segment ((pinned_plug (oldest_pin())), ephemeral_heap_segment)))
20512 size_t entry = deque_pinned_plug();
20513 mark* m = pinned_plug_of (entry);
20514 uint8_t* plug = pinned_plug (m);
20515 size_t len = pinned_len (m);
20516 // detect pinned block in different segment (later) than
20517 // allocation segment, skip those until the oldest pin is in the ephemeral seg.
20518 // adjust the allocation segment along the way (at the end it will
20519 // be the ephemeral segment.
20520 heap_segment* nseg = heap_segment_in_range (generation_allocation_segment (consing_gen));
20522 PREFIX_ASSUME(nseg != NULL);
20524 while (!((plug >= generation_allocation_pointer (consing_gen))&&
20525 (plug < heap_segment_allocated (nseg))))
20527 //adjust the end of the segment to be the end of the plug
20528 assert (generation_allocation_pointer (consing_gen)>=
20529 heap_segment_mem (nseg));
20530 assert (generation_allocation_pointer (consing_gen)<=
20531 heap_segment_committed (nseg));
20533 heap_segment_plan_allocated (nseg) =
20534 generation_allocation_pointer (consing_gen);
20535 //switch allocation segment
20536 nseg = heap_segment_next_rw (nseg);
20537 generation_allocation_segment (consing_gen) = nseg;
20538 //reset the allocation pointer and limits
20539 generation_allocation_pointer (consing_gen) =
20540 heap_segment_mem (nseg);
20542 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
20543 assert(pinned_len(m) == 0 || pinned_len(m) >= Align(min_obj_size));
20544 generation_allocation_pointer (consing_gen) = plug + len;
20545 generation_allocation_limit (consing_gen) =
20546 generation_allocation_pointer (consing_gen);
20548 allocate_in_condemned = TRUE;
20549 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
20552 if (active_new_gen_number != max_generation)
20554 if (active_new_gen_number == (max_generation - 1))
20556 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
20557 if (!demote_gen1_p)
20558 advance_pins_for_demotion (consing_gen);
20561 plan_generation_start (generation_of (active_new_gen_number), consing_gen, x);
20563 dprintf (1, ("process eph: allocated gen%d start at %Ix",
20564 active_new_gen_number,
20565 generation_plan_allocation_start (generation_of (active_new_gen_number))));
20567 if ((demotion_low == MAX_PTR) && !pinned_plug_que_empty_p())
20569 uint8_t* pplug = pinned_plug (oldest_pin());
20570 if (object_gennum (pplug) > 0)
20572 demotion_low = pplug;
20573 dprintf (3, ("process eph: dlow->%Ix", demotion_low));
20577 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
20585 void gc_heap::seg_clear_mark_bits (heap_segment* seg)
20587 uint8_t* o = heap_segment_mem (seg);
20588 while (o < heap_segment_allocated (seg))
20594 o = o + Align (size (o));
20598 #ifdef FEATURE_BASICFREEZE
20599 void gc_heap::sweep_ro_segments (heap_segment* start_seg)
20601 //go through all of the segment in range and reset the mark bit
20602 //TODO works only on small object segments
20604 heap_segment* seg = start_seg;
20608 if (heap_segment_read_only_p (seg) &&
20609 heap_segment_in_range_p (seg))
20611 #ifdef BACKGROUND_GC
20612 if (settings.concurrent)
20614 seg_clear_mark_array_bits_soh (seg);
20618 seg_clear_mark_bits (seg);
20620 #else //BACKGROUND_GC
20623 if(gc_can_use_concurrent)
20625 clear_mark_array (max (heap_segment_mem (seg), lowest_address),
20626 min (heap_segment_allocated (seg), highest_address),
20627 FALSE); // read_only segments need the mark clear
20630 seg_clear_mark_bits (seg);
20631 #endif //MARK_ARRAY
20633 #endif //BACKGROUND_GC
20635 seg = heap_segment_next (seg);
20638 #endif // FEATURE_BASICFREEZE
20640 #ifdef FEATURE_LOH_COMPACTION
20642 BOOL gc_heap::loh_pinned_plug_que_empty_p()
20644 return (loh_pinned_queue_bos == loh_pinned_queue_tos);
20647 void gc_heap::loh_set_allocator_next_pin()
20649 if (!(loh_pinned_plug_que_empty_p()))
20651 mark* oldest_entry = loh_oldest_pin();
20652 uint8_t* plug = pinned_plug (oldest_entry);
20653 generation* gen = large_object_generation;
20654 if ((plug >= generation_allocation_pointer (gen)) &&
20655 (plug < generation_allocation_limit (gen)))
20657 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
20660 assert (!((plug < generation_allocation_pointer (gen)) &&
20661 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
20665 size_t gc_heap::loh_deque_pinned_plug ()
20667 size_t m = loh_pinned_queue_bos;
20668 loh_pinned_queue_bos++;
20673 mark* gc_heap::loh_pinned_plug_of (size_t bos)
20675 return &loh_pinned_queue[bos];
20679 mark* gc_heap::loh_oldest_pin()
20681 return loh_pinned_plug_of (loh_pinned_queue_bos);
20684 // If we can't grow the queue, then don't compact.
20685 BOOL gc_heap::loh_enque_pinned_plug (uint8_t* plug, size_t len)
20687 assert(len >= Align(min_obj_size, get_alignment_constant (FALSE)));
20689 if (loh_pinned_queue_length <= loh_pinned_queue_tos)
20691 if (!grow_mark_stack (loh_pinned_queue, loh_pinned_queue_length, LOH_PIN_QUEUE_LENGTH))
20696 dprintf (3, (" P: %Ix(%Id)", plug, len));
20697 mark& m = loh_pinned_queue[loh_pinned_queue_tos];
20700 loh_pinned_queue_tos++;
20701 loh_set_allocator_next_pin();
20706 BOOL gc_heap::loh_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit)
20708 dprintf (1235, ("trying to fit %Id(%Id) between %Ix and %Ix (%Id)",
20710 (2* AlignQword (loh_padding_obj_size) + size),
20713 (alloc_limit - alloc_pointer)));
20715 return ((alloc_pointer + 2* AlignQword (loh_padding_obj_size) + size) <= alloc_limit);
20718 uint8_t* gc_heap::loh_allocate_in_condemned (uint8_t* old_loc, size_t size)
20720 UNREFERENCED_PARAMETER(old_loc);
20722 generation* gen = large_object_generation;
20723 dprintf (1235, ("E: p:%Ix, l:%Ix, s: %Id",
20724 generation_allocation_pointer (gen),
20725 generation_allocation_limit (gen),
20730 heap_segment* seg = generation_allocation_segment (gen);
20731 if (!(loh_size_fit_p (size, generation_allocation_pointer (gen), generation_allocation_limit (gen))))
20733 if ((!(loh_pinned_plug_que_empty_p()) &&
20734 (generation_allocation_limit (gen) ==
20735 pinned_plug (loh_oldest_pin()))))
20737 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
20738 size_t len = pinned_len (m);
20739 uint8_t* plug = pinned_plug (m);
20740 dprintf (1235, ("AIC: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
20741 pinned_len (m) = plug - generation_allocation_pointer (gen);
20742 generation_allocation_pointer (gen) = plug + len;
20744 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20745 loh_set_allocator_next_pin();
20746 dprintf (1235, ("s: p: %Ix, l: %Ix (%Id)",
20747 generation_allocation_pointer (gen),
20748 generation_allocation_limit (gen),
20749 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20754 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
20756 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20757 dprintf (1235, ("l->pa(%Ix)", generation_allocation_limit (gen)));
20761 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
20763 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
20764 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20765 dprintf (1235, ("l->c(%Ix)", generation_allocation_limit (gen)));
20769 if (loh_size_fit_p (size, generation_allocation_pointer (gen), heap_segment_reserved (seg)) &&
20770 (grow_heap_segment (seg, (generation_allocation_pointer (gen) + size + 2* AlignQword (loh_padding_obj_size)))))
20772 dprintf (1235, ("growing seg from %Ix to %Ix\n", heap_segment_committed (seg),
20773 (generation_allocation_pointer (gen) + size)));
20775 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
20776 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20778 dprintf (1235, ("g: p: %Ix, l: %Ix (%Id)",
20779 generation_allocation_pointer (gen),
20780 generation_allocation_limit (gen),
20781 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20785 heap_segment* next_seg = heap_segment_next (seg);
20786 assert (generation_allocation_pointer (gen)>=
20787 heap_segment_mem (seg));
20788 // Verify that all pinned plugs for this segment are consumed
20789 if (!loh_pinned_plug_que_empty_p() &&
20790 ((pinned_plug (loh_oldest_pin()) <
20791 heap_segment_allocated (seg)) &&
20792 (pinned_plug (loh_oldest_pin()) >=
20793 generation_allocation_pointer (gen))))
20795 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
20796 pinned_plug (loh_oldest_pin())));
20797 dprintf (1236, ("queue empty: %d", loh_pinned_plug_que_empty_p()));
20800 assert (generation_allocation_pointer (gen)>=
20801 heap_segment_mem (seg));
20802 assert (generation_allocation_pointer (gen)<=
20803 heap_segment_committed (seg));
20804 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
20808 // for LOH do we want to try starting from the first LOH every time though?
20809 generation_allocation_segment (gen) = next_seg;
20810 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
20811 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
20813 dprintf (1235, ("n: p: %Ix, l: %Ix (%Id)",
20814 generation_allocation_pointer (gen),
20815 generation_allocation_limit (gen),
20816 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20820 dprintf (1, ("We ran out of space compacting, shouldn't happen"));
20826 loh_set_allocator_next_pin();
20828 dprintf (1235, ("r: p: %Ix, l: %Ix (%Id)",
20829 generation_allocation_pointer (gen),
20830 generation_allocation_limit (gen),
20831 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20838 assert (generation_allocation_pointer (gen)>=
20839 heap_segment_mem (generation_allocation_segment (gen)));
20840 uint8_t* result = generation_allocation_pointer (gen);
20841 size_t loh_pad = AlignQword (loh_padding_obj_size);
20843 generation_allocation_pointer (gen) += size + loh_pad;
20844 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
20846 dprintf (1235, ("p: %Ix, l: %Ix (%Id)",
20847 generation_allocation_pointer (gen),
20848 generation_allocation_limit (gen),
20849 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20851 assert (result + loh_pad);
20852 return result + loh_pad;
20856 BOOL gc_heap::should_compact_loh()
20858 return (loh_compaction_always_p || (loh_compaction_mode != loh_compaction_default));
20862 void gc_heap::check_loh_compact_mode (BOOL all_heaps_compacted_p)
20864 if (settings.loh_compaction && (loh_compaction_mode == loh_compaction_once))
20866 if (all_heaps_compacted_p)
20868 // If the compaction mode says to compact once and we are going to compact LOH,
20869 // we need to revert it back to no compaction.
20870 loh_compaction_mode = loh_compaction_default;
20875 BOOL gc_heap::plan_loh()
20877 if (!loh_pinned_queue)
20879 loh_pinned_queue = new (nothrow) (mark [LOH_PIN_QUEUE_LENGTH]);
20880 if (!loh_pinned_queue)
20882 dprintf (1, ("Cannot allocate the LOH pinned queue (%Id bytes), no compaction",
20883 LOH_PIN_QUEUE_LENGTH * sizeof (mark)));
20887 loh_pinned_queue_length = LOH_PIN_QUEUE_LENGTH;
20890 if (heap_number == 0)
20891 loh_pinned_queue_decay = LOH_PIN_DECAY;
20893 loh_pinned_queue_tos = 0;
20894 loh_pinned_queue_bos = 0;
20896 generation* gen = large_object_generation;
20897 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
20898 PREFIX_ASSUME(start_seg != NULL);
20899 heap_segment* seg = start_seg;
20900 uint8_t* o = generation_allocation_start (gen);
20902 dprintf (1235, ("before GC LOH size: %Id, free list: %Id, free obj: %Id\n",
20903 generation_size (max_generation + 1),
20904 generation_free_list_space (gen),
20905 generation_free_obj_space (gen)));
20909 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
20910 seg = heap_segment_next (seg);
20915 //Skip the generation gap object
20916 o = o + AlignQword (size (o));
20917 // We don't need to ever realloc gen3 start so don't touch it.
20918 heap_segment_plan_allocated (seg) = o;
20919 generation_allocation_pointer (gen) = o;
20920 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
20921 generation_allocation_segment (gen) = start_seg;
20923 uint8_t* free_space_start = o;
20924 uint8_t* free_space_end = o;
20925 uint8_t* new_address = 0;
20929 if (o >= heap_segment_allocated (seg))
20931 seg = heap_segment_next (seg);
20937 o = heap_segment_mem (seg);
20942 free_space_end = o;
20943 size_t size = AlignQword (size (o));
20944 dprintf (1235, ("%Ix(%Id) M", o, size));
20948 // We don't clear the pinned bit yet so we can check in
20949 // compact phase how big a free object we should allocate
20950 // in front of the pinned object. We use the reloc address
20951 // field to store this.
20952 if (!loh_enque_pinned_plug (o, size))
20960 new_address = loh_allocate_in_condemned (o, size);
20963 loh_set_node_relocation_distance (o, (new_address - o));
20964 dprintf (1235, ("lobj %Ix-%Ix -> %Ix-%Ix (%Id)", o, (o + size), new_address, (new_address + size), (new_address - o)));
20967 free_space_start = o;
20968 if (o < heap_segment_allocated (seg))
20970 assert (!marked (o));
20975 while (o < heap_segment_allocated (seg) && !marked (o))
20977 dprintf (1235, ("%Ix(%Id) F (%d)", o, AlignQword (size (o)), ((method_table (o) == g_gc_pFreeObjectMethodTable) ? 1 : 0)));
20978 o = o + AlignQword (size (o));
20983 while (!loh_pinned_plug_que_empty_p())
20985 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
20986 size_t len = pinned_len (m);
20987 uint8_t* plug = pinned_plug (m);
20989 // detect pinned block in different segment (later) than
20990 // allocation segment
20991 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (gen));
20993 while ((plug < generation_allocation_pointer (gen)) ||
20994 (plug >= heap_segment_allocated (nseg)))
20996 assert ((plug < heap_segment_mem (nseg)) ||
20997 (plug > heap_segment_reserved (nseg)));
20998 //adjust the end of the segment to be the end of the plug
20999 assert (generation_allocation_pointer (gen)>=
21000 heap_segment_mem (nseg));
21001 assert (generation_allocation_pointer (gen)<=
21002 heap_segment_committed (nseg));
21004 heap_segment_plan_allocated (nseg) =
21005 generation_allocation_pointer (gen);
21006 //switch allocation segment
21007 nseg = heap_segment_next_rw (nseg);
21008 generation_allocation_segment (gen) = nseg;
21009 //reset the allocation pointer and limits
21010 generation_allocation_pointer (gen) =
21011 heap_segment_mem (nseg);
21014 dprintf (1235, ("SP: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
21015 pinned_len (m) = plug - generation_allocation_pointer (gen);
21016 generation_allocation_pointer (gen) = plug + len;
21019 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
21020 generation_allocation_pointer (gen) = 0;
21021 generation_allocation_limit (gen) = 0;
21026 void gc_heap::compact_loh()
21028 assert (should_compact_loh());
21030 generation* gen = large_object_generation;
21031 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
21032 PREFIX_ASSUME(start_seg != NULL);
21033 heap_segment* seg = start_seg;
21034 heap_segment* prev_seg = 0;
21035 uint8_t* o = generation_allocation_start (gen);
21037 //Skip the generation gap object
21038 o = o + AlignQword (size (o));
21039 // We don't need to ever realloc gen3 start so don't touch it.
21040 uint8_t* free_space_start = o;
21041 uint8_t* free_space_end = o;
21042 generation_allocator (gen)->clear();
21043 generation_free_list_space (gen) = 0;
21044 generation_free_obj_space (gen) = 0;
21046 loh_pinned_queue_bos = 0;
21050 if (o >= heap_segment_allocated (seg))
21052 heap_segment* next_seg = heap_segment_next (seg);
21054 if ((heap_segment_plan_allocated (seg) == heap_segment_mem (seg)) &&
21055 (seg != start_seg) && !heap_segment_read_only_p (seg))
21057 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
21059 heap_segment_next (prev_seg) = next_seg;
21060 heap_segment_next (seg) = freeable_large_heap_segment;
21061 freeable_large_heap_segment = seg;
21065 if (!heap_segment_read_only_p (seg))
21067 // We grew the segment to accommodate allocations.
21068 if (heap_segment_plan_allocated (seg) > heap_segment_allocated (seg))
21070 if ((heap_segment_plan_allocated (seg) - plug_skew) > heap_segment_used (seg))
21072 heap_segment_used (seg) = heap_segment_plan_allocated (seg) - plug_skew;
21076 heap_segment_allocated (seg) = heap_segment_plan_allocated (seg);
21077 dprintf (3, ("Trimming seg to %Ix[", heap_segment_allocated (seg)));
21078 decommit_heap_segment_pages (seg, 0);
21079 dprintf (1236, ("CLOH: seg: %Ix, alloc: %Ix, used: %Ix, committed: %Ix",
21081 heap_segment_allocated (seg),
21082 heap_segment_used (seg),
21083 heap_segment_committed (seg)));
21084 //heap_segment_used (seg) = heap_segment_allocated (seg) - plug_skew;
21085 dprintf (1236, ("CLOH: used is set to %Ix", heap_segment_used (seg)));
21095 o = heap_segment_mem (seg);
21101 free_space_end = o;
21102 size_t size = AlignQword (size (o));
21105 uint8_t* reloc = o;
21110 // We are relying on the fact the pinned objects are always looked at in the same order
21111 // in plan phase and in compact phase.
21112 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21113 uint8_t* plug = pinned_plug (m);
21114 assert (plug == o);
21116 loh_pad = pinned_len (m);
21121 loh_pad = AlignQword (loh_padding_obj_size);
21123 reloc += loh_node_relocation_distance (o);
21124 gcmemcopy (reloc, o, size, TRUE);
21127 thread_gap ((reloc - loh_pad), loh_pad, gen);
21130 free_space_start = o;
21131 if (o < heap_segment_allocated (seg))
21133 assert (!marked (o));
21138 while (o < heap_segment_allocated (seg) && !marked (o))
21140 o = o + AlignQword (size (o));
21145 assert (loh_pinned_plug_que_empty_p());
21147 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21148 generation_size (max_generation + 1),
21149 generation_free_list_space (gen),
21150 generation_free_obj_space (gen)));
21153 void gc_heap::relocate_in_loh_compact()
21155 generation* gen = large_object_generation;
21156 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21157 uint8_t* o = generation_allocation_start (gen);
21159 //Skip the generation gap object
21160 o = o + AlignQword (size (o));
21162 relocate_args args;
21164 args.high = gc_high;
21165 args.last_plug = 0;
21169 if (o >= heap_segment_allocated (seg))
21171 seg = heap_segment_next (seg);
21177 o = heap_segment_mem (seg);
21182 size_t size = AlignQword (size (o));
21184 check_class_object_demotion (o);
21185 if (contain_pointers (o))
21187 go_through_object_nostart (method_table (o), o, size(o), pval,
21189 reloc_survivor_helper (pval);
21194 if (o < heap_segment_allocated (seg))
21196 assert (!marked (o));
21201 while (o < heap_segment_allocated (seg) && !marked (o))
21203 o = o + AlignQword (size (o));
21208 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21209 generation_size (max_generation + 1),
21210 generation_free_list_space (gen),
21211 generation_free_obj_space (gen)));
21214 void gc_heap::walk_relocation_for_loh (void* profiling_context, record_surv_fn fn)
21216 generation* gen = large_object_generation;
21217 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21218 uint8_t* o = generation_allocation_start (gen);
21220 //Skip the generation gap object
21221 o = o + AlignQword (size (o));
21225 if (o >= heap_segment_allocated (seg))
21227 seg = heap_segment_next (seg);
21233 o = heap_segment_mem (seg);
21238 size_t size = AlignQword (size (o));
21240 ptrdiff_t reloc = loh_node_relocation_distance (o);
21242 STRESS_LOG_PLUG_MOVE(o, (o + size), -reloc);
21244 fn (o, (o + size), reloc, profiling_context, !!settings.compaction, false);
21247 if (o < heap_segment_allocated (seg))
21249 assert (!marked (o));
21254 while (o < heap_segment_allocated (seg) && !marked (o))
21256 o = o + AlignQword (size (o));
21262 BOOL gc_heap::loh_object_p (uint8_t* o)
21264 #ifdef MULTIPLE_HEAPS
21265 gc_heap* hp = gc_heap::g_heaps [0];
21266 int brick_entry = hp->brick_table[hp->brick_of (o)];
21267 #else //MULTIPLE_HEAPS
21268 int brick_entry = brick_table[brick_of (o)];
21269 #endif //MULTIPLE_HEAPS
21271 return (brick_entry == 0);
21273 #endif //FEATURE_LOH_COMPACTION
21275 void gc_heap::convert_to_pinned_plug (BOOL& last_npinned_plug_p,
21276 BOOL& last_pinned_plug_p,
21277 BOOL& pinned_plug_p,
21279 size_t& artificial_pinned_size)
21281 last_npinned_plug_p = FALSE;
21282 last_pinned_plug_p = TRUE;
21283 pinned_plug_p = TRUE;
21284 artificial_pinned_size = ps;
21287 // Because we have the artificial pinning, we can't guarantee that pinned and npinned
21288 // plugs are always interleaved.
21289 void gc_heap::store_plug_gap_info (uint8_t* plug_start,
21291 BOOL& last_npinned_plug_p,
21292 BOOL& last_pinned_plug_p,
21293 uint8_t*& last_pinned_plug,
21294 BOOL& pinned_plug_p,
21295 uint8_t* last_object_in_last_plug,
21296 BOOL& merge_with_last_pin_p,
21297 // this is only for verification purpose
21298 size_t last_plug_len)
21300 UNREFERENCED_PARAMETER(last_plug_len);
21302 if (!last_npinned_plug_p && !last_pinned_plug_p)
21304 //dprintf (3, ("last full plug end: %Ix, full plug start: %Ix", plug_end, plug_start));
21305 dprintf (3, ("Free: %Ix", (plug_start - plug_end)));
21306 assert ((plug_start == plug_end) || ((size_t)(plug_start - plug_end) >= Align (min_obj_size)));
21307 set_gap_size (plug_start, plug_start - plug_end);
21310 if (pinned (plug_start))
21312 BOOL save_pre_plug_info_p = FALSE;
21314 if (last_npinned_plug_p || last_pinned_plug_p)
21316 //if (last_plug_len == Align (min_obj_size))
21318 // dprintf (3, ("debugging only - last npinned plug is min, check to see if it's correct"));
21319 // GCToOSInterface::DebugBreak();
21321 save_pre_plug_info_p = TRUE;
21324 pinned_plug_p = TRUE;
21325 last_npinned_plug_p = FALSE;
21327 if (last_pinned_plug_p)
21329 dprintf (3, ("last plug %Ix was also pinned, should merge", last_pinned_plug));
21330 merge_with_last_pin_p = TRUE;
21334 last_pinned_plug_p = TRUE;
21335 last_pinned_plug = plug_start;
21337 enque_pinned_plug (last_pinned_plug, save_pre_plug_info_p, last_object_in_last_plug);
21339 if (save_pre_plug_info_p)
21341 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21347 if (last_pinned_plug_p)
21349 //if (Align (last_plug_len) < min_pre_pin_obj_size)
21351 // dprintf (3, ("debugging only - last pinned plug is min, check to see if it's correct"));
21352 // GCToOSInterface::DebugBreak();
21355 save_post_plug_info (last_pinned_plug, last_object_in_last_plug, plug_start);
21356 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21358 verify_pins_with_post_plug_info("after saving post plug info");
21360 last_npinned_plug_p = TRUE;
21361 last_pinned_plug_p = FALSE;
21365 void gc_heap::record_interesting_data_point (interesting_data_point idp)
21367 #ifdef GC_CONFIG_DRIVEN
21368 (interesting_data_per_gc[idp])++;
21370 UNREFERENCED_PARAMETER(idp);
21371 #endif //GC_CONFIG_DRIVEN
21375 #pragma warning(push)
21376 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
21378 void gc_heap::plan_phase (int condemned_gen_number)
21380 size_t old_gen2_allocated = 0;
21381 size_t old_gen2_size = 0;
21383 if (condemned_gen_number == (max_generation - 1))
21385 old_gen2_allocated = generation_free_list_allocated (generation_of (max_generation));
21386 old_gen2_size = generation_size (max_generation);
21389 assert (settings.concurrent == FALSE);
21391 // %type% category = quote (plan);
21395 start = GetCycleCount32();
21398 dprintf (2,("---- Plan Phase ---- Condemned generation %d, promotion: %d",
21399 condemned_gen_number, settings.promotion ? 1 : 0));
21401 generation* condemned_gen1 = generation_of (condemned_gen_number);
21404 BOOL use_mark_list = FALSE;
21405 uint8_t** mark_list_next = &mark_list[0];
21406 #ifdef GC_CONFIG_DRIVEN
21407 dprintf (3, ("total number of marked objects: %Id (%Id)",
21408 (mark_list_index - &mark_list[0]), ((mark_list_end - &mark_list[0]))));
21410 dprintf (3, ("mark_list length: %Id",
21411 (mark_list_index - &mark_list[0])));
21412 #endif //GC_CONFIG_DRIVEN
21414 if ((condemned_gen_number < max_generation) &&
21415 (mark_list_index <= mark_list_end)
21416 #ifdef BACKGROUND_GC
21417 && (!recursive_gc_sync::background_running_p())
21418 #endif //BACKGROUND_GC
21421 #ifndef MULTIPLE_HEAPS
21422 _sort (&mark_list[0], mark_list_index-1, 0);
21423 //printf ("using mark list at GC #%d", dd_collection_count (dynamic_data_of (0)));
21424 //verify_qsort_array (&mark_list[0], mark_list_index-1);
21425 #endif //!MULTIPLE_HEAPS
21426 use_mark_list = TRUE;
21427 get_gc_data_per_heap()->set_mechanism_bit (gc_mark_list_bit);
21431 dprintf (3, ("mark_list not used"));
21436 #ifdef FEATURE_BASICFREEZE
21437 if ((generation_start_segment (condemned_gen1) != ephemeral_heap_segment) &&
21438 ro_segments_in_range)
21440 sweep_ro_segments (generation_start_segment (condemned_gen1));
21442 #endif // FEATURE_BASICFREEZE
21444 #ifndef MULTIPLE_HEAPS
21445 if (shigh != (uint8_t*)0)
21447 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
21449 PREFIX_ASSUME(seg != NULL);
21451 heap_segment* fseg = seg;
21454 if (slow > heap_segment_mem (seg) &&
21455 slow < heap_segment_reserved (seg))
21459 uint8_t* o = generation_allocation_start (condemned_gen1) +
21460 Align (size (generation_allocation_start (condemned_gen1)));
21463 assert ((slow - o) >= (int)Align (min_obj_size));
21464 #ifdef BACKGROUND_GC
21465 if (current_c_gc_state == c_gc_state_marking)
21467 bgc_clear_batch_mark_array_bits (o, slow);
21469 #endif //BACKGROUND_GC
21470 make_unused_array (o, slow - o);
21475 assert (condemned_gen_number == max_generation);
21476 make_unused_array (heap_segment_mem (seg),
21477 slow - heap_segment_mem (seg));
21480 if (in_range_for_segment (shigh, seg))
21482 #ifdef BACKGROUND_GC
21483 if (current_c_gc_state == c_gc_state_marking)
21485 bgc_clear_batch_mark_array_bits ((shigh + Align (size (shigh))), heap_segment_allocated (seg));
21487 #endif //BACKGROUND_GC
21488 heap_segment_allocated (seg) = shigh + Align (size (shigh));
21490 // test if the segment is in the range of [slow, shigh]
21491 if (!((heap_segment_reserved (seg) >= slow) &&
21492 (heap_segment_mem (seg) <= shigh)))
21494 // shorten it to minimum
21495 heap_segment_allocated (seg) = heap_segment_mem (seg);
21497 seg = heap_segment_next_rw (seg);
21502 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
21504 PREFIX_ASSUME(seg != NULL);
21506 heap_segment* sseg = seg;
21509 // shorten it to minimum
21512 // no survivors make all generations look empty
21513 uint8_t* o = generation_allocation_start (condemned_gen1) +
21514 Align (size (generation_allocation_start (condemned_gen1)));
21515 #ifdef BACKGROUND_GC
21516 if (current_c_gc_state == c_gc_state_marking)
21518 bgc_clear_batch_mark_array_bits (o, heap_segment_allocated (seg));
21520 #endif //BACKGROUND_GC
21521 heap_segment_allocated (seg) = o;
21525 assert (condemned_gen_number == max_generation);
21526 #ifdef BACKGROUND_GC
21527 if (current_c_gc_state == c_gc_state_marking)
21529 bgc_clear_batch_mark_array_bits (heap_segment_mem (seg), heap_segment_allocated (seg));
21531 #endif //BACKGROUND_GC
21532 heap_segment_allocated (seg) = heap_segment_mem (seg);
21534 seg = heap_segment_next_rw (seg);
21538 #endif //MULTIPLE_HEAPS
21540 heap_segment* seg1 = heap_segment_rw (generation_start_segment (condemned_gen1));
21542 PREFIX_ASSUME(seg1 != NULL);
21544 uint8_t* end = heap_segment_allocated (seg1);
21545 uint8_t* first_condemned_address = generation_allocation_start (condemned_gen1);
21546 uint8_t* x = first_condemned_address;
21548 assert (!marked (x));
21549 uint8_t* plug_end = x;
21551 size_t sequence_number = 0;
21552 uint8_t* last_node = 0;
21553 size_t current_brick = brick_of (x);
21554 BOOL allocate_in_condemned = ((condemned_gen_number == max_generation)||
21555 (settings.promotion == FALSE));
21556 int active_old_gen_number = condemned_gen_number;
21557 int active_new_gen_number = (allocate_in_condemned ? condemned_gen_number:
21558 (1 + condemned_gen_number));
21559 generation* older_gen = 0;
21560 generation* consing_gen = condemned_gen1;
21561 alloc_list r_free_list [MAX_BUCKET_COUNT];
21563 size_t r_free_list_space = 0;
21564 size_t r_free_obj_space = 0;
21565 size_t r_older_gen_free_list_allocated = 0;
21566 size_t r_older_gen_condemned_allocated = 0;
21567 size_t r_older_gen_end_seg_allocated = 0;
21568 uint8_t* r_allocation_pointer = 0;
21569 uint8_t* r_allocation_limit = 0;
21570 uint8_t* r_allocation_start_region = 0;
21571 heap_segment* r_allocation_segment = 0;
21572 #ifdef FREE_USAGE_STATS
21573 size_t r_older_gen_free_space[NUM_GEN_POWER2];
21574 #endif //FREE_USAGE_STATS
21576 if ((condemned_gen_number < max_generation))
21578 older_gen = generation_of (min (max_generation, 1 + condemned_gen_number));
21579 generation_allocator (older_gen)->copy_to_alloc_list (r_free_list);
21581 r_free_list_space = generation_free_list_space (older_gen);
21582 r_free_obj_space = generation_free_obj_space (older_gen);
21583 #ifdef FREE_USAGE_STATS
21584 memcpy (r_older_gen_free_space, older_gen->gen_free_spaces, sizeof (r_older_gen_free_space));
21585 #endif //FREE_USAGE_STATS
21586 generation_allocate_end_seg_p (older_gen) = FALSE;
21587 r_older_gen_free_list_allocated = generation_free_list_allocated (older_gen);
21588 r_older_gen_condemned_allocated = generation_condemned_allocated (older_gen);
21589 r_older_gen_end_seg_allocated = generation_end_seg_allocated (older_gen);
21590 r_allocation_limit = generation_allocation_limit (older_gen);
21591 r_allocation_pointer = generation_allocation_pointer (older_gen);
21592 r_allocation_start_region = generation_allocation_context_start_region (older_gen);
21593 r_allocation_segment = generation_allocation_segment (older_gen);
21594 heap_segment* start_seg = heap_segment_rw (generation_start_segment (older_gen));
21596 PREFIX_ASSUME(start_seg != NULL);
21598 if (start_seg != ephemeral_heap_segment)
21600 assert (condemned_gen_number == (max_generation - 1));
21601 while (start_seg && (start_seg != ephemeral_heap_segment))
21603 assert (heap_segment_allocated (start_seg) >=
21604 heap_segment_mem (start_seg));
21605 assert (heap_segment_allocated (start_seg) <=
21606 heap_segment_reserved (start_seg));
21607 heap_segment_plan_allocated (start_seg) =
21608 heap_segment_allocated (start_seg);
21609 start_seg = heap_segment_next_rw (start_seg);
21614 //reset all of the segment allocated sizes
21616 heap_segment* seg2 = heap_segment_rw (generation_start_segment (condemned_gen1));
21618 PREFIX_ASSUME(seg2 != NULL);
21622 heap_segment_plan_allocated (seg2) =
21623 heap_segment_mem (seg2);
21624 seg2 = heap_segment_next_rw (seg2);
21627 int condemned_gn = condemned_gen_number;
21629 int bottom_gen = 0;
21630 init_free_and_plug();
21632 while (condemned_gn >= bottom_gen)
21634 generation* condemned_gen2 = generation_of (condemned_gn);
21635 generation_allocator (condemned_gen2)->clear();
21636 generation_free_list_space (condemned_gen2) = 0;
21637 generation_free_obj_space (condemned_gen2) = 0;
21638 generation_allocation_size (condemned_gen2) = 0;
21639 generation_condemned_allocated (condemned_gen2) = 0;
21640 generation_pinned_allocated (condemned_gen2) = 0;
21641 generation_free_list_allocated(condemned_gen2) = 0;
21642 generation_end_seg_allocated (condemned_gen2) = 0;
21643 generation_pinned_allocation_sweep_size (condemned_gen2) = 0;
21644 generation_pinned_allocation_compact_size (condemned_gen2) = 0;
21645 #ifdef FREE_USAGE_STATS
21646 generation_pinned_free_obj_space (condemned_gen2) = 0;
21647 generation_allocated_in_pinned_free (condemned_gen2) = 0;
21648 generation_allocated_since_last_pin (condemned_gen2) = 0;
21649 #endif //FREE_USAGE_STATS
21650 generation_plan_allocation_start (condemned_gen2) = 0;
21651 generation_allocation_segment (condemned_gen2) =
21652 heap_segment_rw (generation_start_segment (condemned_gen2));
21654 PREFIX_ASSUME(generation_allocation_segment(condemned_gen2) != NULL);
21656 if (generation_start_segment (condemned_gen2) != ephemeral_heap_segment)
21658 generation_allocation_pointer (condemned_gen2) =
21659 heap_segment_mem (generation_allocation_segment (condemned_gen2));
21663 generation_allocation_pointer (condemned_gen2) = generation_allocation_start (condemned_gen2);
21666 generation_allocation_limit (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
21667 generation_allocation_context_start_region (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
21672 BOOL allocate_first_generation_start = FALSE;
21674 if (allocate_in_condemned)
21676 allocate_first_generation_start = TRUE;
21679 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
21681 demotion_low = MAX_PTR;
21682 demotion_high = heap_segment_allocated (ephemeral_heap_segment);
21684 // If we are doing a gen1 only because of cards, it means we should not demote any pinned plugs
21685 // from gen1. They should get promoted to gen2.
21686 demote_gen1_p = !(settings.promotion &&
21687 (settings.condemned_generation == (max_generation - 1)) &&
21688 gen_to_condemn_reasons.is_only_condition (gen_low_card_p));
21690 total_ephemeral_size = 0;
21692 print_free_and_plug ("BP");
21694 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
21696 generation* temp_gen = generation_of (gen_idx);
21698 dprintf (2, ("gen%d start %Ix, plan start %Ix",
21700 generation_allocation_start (temp_gen),
21701 generation_plan_allocation_start (temp_gen)));
21704 BOOL fire_pinned_plug_events_p = ETW_EVENT_ENABLED(MICROSOFT_WINDOWS_DOTNETRUNTIME_PRIVATE_PROVIDER_Context, PinPlugAtGCTime);
21705 size_t last_plug_len = 0;
21712 assert (heap_segment_allocated (seg1) == end);
21713 heap_segment_allocated (seg1) = plug_end;
21715 current_brick = update_brick_table (tree, current_brick, x, plug_end);
21716 dprintf (3, ("end of seg: new tree, sequence# 0"));
21717 sequence_number = 0;
21720 if (heap_segment_next_rw (seg1))
21722 seg1 = heap_segment_next_rw (seg1);
21723 end = heap_segment_allocated (seg1);
21724 plug_end = x = heap_segment_mem (seg1);
21725 current_brick = brick_of (x);
21726 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
21735 BOOL last_npinned_plug_p = FALSE;
21736 BOOL last_pinned_plug_p = FALSE;
21738 // last_pinned_plug is the beginning of the last pinned plug. If we merge a plug into a pinned
21739 // plug we do not change the value of last_pinned_plug. This happens with artificially pinned plugs -
21740 // it can be merged with a previous pinned plug and a pinned plug after it can be merged with it.
21741 uint8_t* last_pinned_plug = 0;
21742 size_t num_pinned_plugs_in_plug = 0;
21744 uint8_t* last_object_in_plug = 0;
21746 while ((x < end) && marked (x))
21748 uint8_t* plug_start = x;
21749 uint8_t* saved_plug_end = plug_end;
21750 BOOL pinned_plug_p = FALSE;
21751 BOOL npin_before_pin_p = FALSE;
21752 BOOL saved_last_npinned_plug_p = last_npinned_plug_p;
21753 uint8_t* saved_last_object_in_plug = last_object_in_plug;
21754 BOOL merge_with_last_pin_p = FALSE;
21756 size_t added_pinning_size = 0;
21757 size_t artificial_pinned_size = 0;
21759 store_plug_gap_info (plug_start, plug_end, last_npinned_plug_p, last_pinned_plug_p,
21760 last_pinned_plug, pinned_plug_p, last_object_in_plug,
21761 merge_with_last_pin_p, last_plug_len);
21763 #ifdef FEATURE_STRUCTALIGN
21764 int requiredAlignment = ((CObjectHeader*)plug_start)->GetRequiredAlignment();
21765 size_t alignmentOffset = OBJECT_ALIGNMENT_OFFSET;
21766 #endif // FEATURE_STRUCTALIGN
21770 while ((xl < end) && marked (xl) && (pinned (xl) == pinned_plug_p))
21777 #ifdef FEATURE_STRUCTALIGN
21780 int obj_requiredAlignment = ((CObjectHeader*)xl)->GetRequiredAlignment();
21781 if (obj_requiredAlignment > requiredAlignment)
21783 requiredAlignment = obj_requiredAlignment;
21784 alignmentOffset = xl - plug_start + OBJECT_ALIGNMENT_OFFSET;
21787 #endif // FEATURE_STRUCTALIGN
21791 dprintf(4, ("+%Ix+", (size_t)xl));
21792 assert ((size (xl) > 0));
21793 assert ((size (xl) <= LARGE_OBJECT_SIZE));
21795 last_object_in_plug = xl;
21797 xl = xl + Align (size (xl));
21801 BOOL next_object_marked_p = ((xl < end) && marked (xl));
21805 // If it is pinned we need to extend to the next marked object as we can't use part of
21806 // a pinned object to make the artificial gap (unless the last 3 ptr sized words are all
21807 // references but for now I am just using the next non pinned object for that).
21808 if (next_object_marked_p)
21811 last_object_in_plug = xl;
21812 size_t extra_size = Align (size (xl));
21813 xl = xl + extra_size;
21814 added_pinning_size = extra_size;
21819 if (next_object_marked_p)
21820 npin_before_pin_p = TRUE;
21823 assert (xl <= end);
21826 dprintf (3, ( "%Ix[", (size_t)x));
21828 size_t ps = plug_end - plug_start;
21829 last_plug_len = ps;
21830 dprintf (3, ( "%Ix[(%Ix)", (size_t)x, ps));
21831 uint8_t* new_address = 0;
21833 if (!pinned_plug_p)
21835 if (allocate_in_condemned &&
21836 (settings.condemned_generation == max_generation) &&
21837 (ps > OS_PAGE_SIZE))
21839 ptrdiff_t reloc = plug_start - generation_allocation_pointer (consing_gen);
21840 //reloc should >=0 except when we relocate
21841 //across segments and the dest seg is higher then the src
21843 if ((ps > (8*OS_PAGE_SIZE)) &&
21845 ((size_t)reloc < (ps/16)))
21847 dprintf (3, ("Pinning %Ix; reloc would have been: %Ix",
21848 (size_t)plug_start, reloc));
21849 // The last plug couldn't have been a npinned plug or it would have
21850 // included this plug.
21851 assert (!saved_last_npinned_plug_p);
21853 if (last_pinned_plug)
21855 dprintf (3, ("artificially pinned plug merged with last pinned plug"));
21856 merge_with_last_pin_p = TRUE;
21860 enque_pinned_plug (plug_start, FALSE, 0);
21861 last_pinned_plug = plug_start;
21864 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
21865 ps, artificial_pinned_size);
21870 if (allocate_first_generation_start)
21872 allocate_first_generation_start = FALSE;
21873 plan_generation_start (condemned_gen1, consing_gen, plug_start);
21874 assert (generation_plan_allocation_start (condemned_gen1));
21877 if (seg1 == ephemeral_heap_segment)
21879 process_ephemeral_boundaries (plug_start, active_new_gen_number,
21880 active_old_gen_number,
21882 allocate_in_condemned);
21885 dprintf (3, ("adding %Id to gen%d surv", ps, active_old_gen_number));
21887 dynamic_data* dd_active_old = dynamic_data_of (active_old_gen_number);
21888 dd_survived_size (dd_active_old) += ps;
21890 BOOL convert_to_pinned_p = FALSE;
21892 if (!pinned_plug_p)
21894 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
21895 dd_num_npinned_plugs (dd_active_old)++;
21896 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
21898 add_gen_plug (active_old_gen_number, ps);
21900 if (allocate_in_condemned)
21902 verify_pins_with_post_plug_info("before aic");
21905 allocate_in_condemned_generations (consing_gen,
21907 active_old_gen_number,
21909 &convert_to_pinned_p,
21910 (npin_before_pin_p ? plug_end : 0),
21912 #endif //SHORT_PLUGS
21913 plug_start REQD_ALIGN_AND_OFFSET_ARG);
21914 verify_pins_with_post_plug_info("after aic");
21918 new_address = allocate_in_older_generation (older_gen, ps, active_old_gen_number, plug_start REQD_ALIGN_AND_OFFSET_ARG);
21920 if (new_address != 0)
21922 if (settings.condemned_generation == (max_generation - 1))
21924 dprintf (3, (" NA: %Ix-%Ix -> %Ix, %Ix (%Ix)",
21925 plug_start, plug_end,
21926 (size_t)new_address, (size_t)new_address + (plug_end - plug_start),
21927 (size_t)(plug_end - plug_start)));
21932 allocate_in_condemned = TRUE;
21934 new_address = allocate_in_condemned_generations (consing_gen, ps, active_old_gen_number,
21936 &convert_to_pinned_p,
21937 (npin_before_pin_p ? plug_end : 0),
21939 #endif //SHORT_PLUGS
21940 plug_start REQD_ALIGN_AND_OFFSET_ARG);
21944 if (convert_to_pinned_p)
21946 assert (last_npinned_plug_p != FALSE);
21947 assert (last_pinned_plug_p == FALSE);
21948 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
21949 ps, artificial_pinned_size);
21950 enque_pinned_plug (plug_start, FALSE, 0);
21951 last_pinned_plug = plug_start;
21957 //verify that we are at then end of the ephemeral segment
21958 assert (generation_allocation_segment (consing_gen) ==
21959 ephemeral_heap_segment);
21960 //verify that we are near the end
21961 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) <
21962 heap_segment_allocated (ephemeral_heap_segment));
21963 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) >
21964 (heap_segment_allocated (ephemeral_heap_segment) + Align (min_obj_size)));
21968 #ifdef SIMPLE_DPRINTF
21969 dprintf (3, ("(%Ix)[%Ix->%Ix, NA: [%Ix(%Id), %Ix[: %Ix(%d)",
21970 (size_t)(node_gap_size (plug_start)),
21971 plug_start, plug_end, (size_t)new_address, (size_t)(plug_start - new_address),
21972 (size_t)new_address + ps, ps,
21973 (is_plug_padded (plug_start) ? 1 : 0)));
21974 #endif //SIMPLE_DPRINTF
21977 if (is_plug_padded (plug_start))
21979 dprintf (3, ("%Ix was padded", plug_start));
21980 dd_padding_size (dd_active_old) += Align (min_obj_size);
21982 #endif //SHORT_PLUGS
21989 if (fire_pinned_plug_events_p)
21990 FireEtwPinPlugAtGCTime(plug_start, plug_end,
21991 (merge_with_last_pin_p ? 0 : (uint8_t*)node_gap_size (plug_start)),
21992 GetClrInstanceId());
21994 if (merge_with_last_pin_p)
21996 merge_with_last_pinned_plug (last_pinned_plug, ps);
22000 assert (last_pinned_plug == plug_start);
22001 set_pinned_info (plug_start, ps, consing_gen);
22004 new_address = plug_start;
22006 dprintf (3, ( "(%Ix)PP: [%Ix, %Ix[%Ix](m:%d)",
22007 (size_t)(node_gap_size (plug_start)), (size_t)plug_start,
22008 (size_t)plug_end, ps,
22009 (merge_with_last_pin_p ? 1 : 0)));
22011 dprintf (3, ("adding %Id to gen%d pinned surv", plug_end - plug_start, active_old_gen_number));
22012 dd_pinned_survived_size (dd_active_old) += plug_end - plug_start;
22013 dd_added_pinned_size (dd_active_old) += added_pinning_size;
22014 dd_artificial_pinned_survived_size (dd_active_old) += artificial_pinned_size;
22016 if (!demote_gen1_p && (active_old_gen_number == (max_generation - 1)))
22018 last_gen1_pin_end = plug_end;
22023 // detect forward allocation in the same segment
22024 assert (!((new_address > plug_start) &&
22025 (new_address < heap_segment_reserved (seg1))));
22028 if (!merge_with_last_pin_p)
22030 if (current_brick != brick_of (plug_start))
22032 current_brick = update_brick_table (tree, current_brick, plug_start, saved_plug_end);
22033 sequence_number = 0;
22037 set_node_relocation_distance (plug_start, (new_address - plug_start));
22038 if (last_node && (node_relocation_distance (last_node) ==
22039 (node_relocation_distance (plug_start) +
22040 node_gap_size (plug_start))))
22042 //dprintf(3,( " Lb"));
22043 dprintf (3, ("%Ix Lb", plug_start));
22044 set_node_left (plug_start);
22046 if (0 == sequence_number)
22048 dprintf (2, ("sn: 0, tree is set to %Ix", plug_start));
22052 verify_pins_with_post_plug_info("before insert node");
22054 tree = insert_node (plug_start, ++sequence_number, tree, last_node);
22055 dprintf (3, ("tree is %Ix (b: %Ix) after insert_node", tree, brick_of (tree)));
22056 last_node = plug_start;
22059 // If we detect if the last plug is pinned plug right before us, we should save this gap info
22060 if (!pinned_plug_p)
22062 if (mark_stack_tos > 0)
22064 mark& m = mark_stack_array[mark_stack_tos - 1];
22065 if (m.has_post_plug_info())
22067 uint8_t* post_plug_info_start = m.saved_post_plug_info_start;
22068 size_t* current_plug_gap_start = (size_t*)(plug_start - sizeof (plug_and_gap));
22069 if ((uint8_t*)current_plug_gap_start == post_plug_info_start)
22071 dprintf (3, ("Ginfo: %Ix, %Ix, %Ix",
22072 *current_plug_gap_start, *(current_plug_gap_start + 1),
22073 *(current_plug_gap_start + 2)));
22074 memcpy (&(m.saved_post_plug_debug), current_plug_gap_start, sizeof (gap_reloc_pair));
22081 verify_pins_with_post_plug_info("after insert node");
22085 if (num_pinned_plugs_in_plug > 1)
22087 dprintf (3, ("more than %Id pinned plugs in this plug", num_pinned_plugs_in_plug));
22094 while ((mark_list_next < mark_list_index) &&
22095 (*mark_list_next <= x))
22099 if ((mark_list_next < mark_list_index)
22100 #ifdef MULTIPLE_HEAPS
22101 && (*mark_list_next < end) //for multiple segments
22102 #endif //MULTIPLE_HEAPS
22104 x = *mark_list_next;
22112 #ifdef BACKGROUND_GC
22113 if (current_c_gc_state == c_gc_state_marking)
22115 assert (recursive_gc_sync::background_running_p());
22116 while ((xl < end) && !marked (xl))
22118 dprintf (4, ("-%Ix-", (size_t)xl));
22119 assert ((size (xl) > 0));
22120 background_object_marked (xl, TRUE);
22121 xl = xl + Align (size (xl));
22126 #endif //BACKGROUND_GC
22128 while ((xl < end) && !marked (xl))
22130 dprintf (4, ("-%Ix-", (size_t)xl));
22131 assert ((size (xl) > 0));
22132 xl = xl + Align (size (xl));
22136 assert (xl <= end);
22142 while (!pinned_plug_que_empty_p())
22144 if (settings.promotion)
22146 uint8_t* pplug = pinned_plug (oldest_pin());
22147 if (in_range_for_segment (pplug, ephemeral_heap_segment))
22149 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
22150 //allocate all of the generation gaps
22151 while (active_new_gen_number > 0)
22153 active_new_gen_number--;
22155 if (active_new_gen_number == (max_generation - 1))
22157 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
22158 if (!demote_gen1_p)
22159 advance_pins_for_demotion (consing_gen);
22162 generation* gen = generation_of (active_new_gen_number);
22163 plan_generation_start (gen, consing_gen, 0);
22165 if (demotion_low == MAX_PTR)
22167 demotion_low = pplug;
22168 dprintf (3, ("end plan: dlow->%Ix", demotion_low));
22171 dprintf (2, ("(%d)gen%d plan start: %Ix",
22172 heap_number, active_new_gen_number, (size_t)generation_plan_allocation_start (gen)));
22173 assert (generation_plan_allocation_start (gen));
22178 if (pinned_plug_que_empty_p())
22181 size_t entry = deque_pinned_plug();
22182 mark* m = pinned_plug_of (entry);
22183 uint8_t* plug = pinned_plug (m);
22184 size_t len = pinned_len (m);
22186 // detect pinned block in different segment (later) than
22187 // allocation segment
22188 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (consing_gen));
22190 while ((plug < generation_allocation_pointer (consing_gen)) ||
22191 (plug >= heap_segment_allocated (nseg)))
22193 assert ((plug < heap_segment_mem (nseg)) ||
22194 (plug > heap_segment_reserved (nseg)));
22195 //adjust the end of the segment to be the end of the plug
22196 assert (generation_allocation_pointer (consing_gen)>=
22197 heap_segment_mem (nseg));
22198 assert (generation_allocation_pointer (consing_gen)<=
22199 heap_segment_committed (nseg));
22201 heap_segment_plan_allocated (nseg) =
22202 generation_allocation_pointer (consing_gen);
22203 //switch allocation segment
22204 nseg = heap_segment_next_rw (nseg);
22205 generation_allocation_segment (consing_gen) = nseg;
22206 //reset the allocation pointer and limits
22207 generation_allocation_pointer (consing_gen) =
22208 heap_segment_mem (nseg);
22211 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
22212 dprintf (2, ("pin %Ix b: %Ix->%Ix", plug, brick_of (plug),
22213 (size_t)(brick_table[brick_of (plug)])));
22215 generation_allocation_pointer (consing_gen) = plug + len;
22216 generation_allocation_limit (consing_gen) =
22217 generation_allocation_pointer (consing_gen);
22218 //Add the size of the pinned plug to the right pinned allocations
22219 //find out which gen this pinned plug came from
22220 int frgn = object_gennum (plug);
22221 if ((frgn != (int)max_generation) && settings.promotion)
22223 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
22228 plan_generation_starts (consing_gen);
22229 print_free_and_plug ("AP");
22232 #ifdef SIMPLE_DPRINTF
22233 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
22235 generation* temp_gen = generation_of (gen_idx);
22236 dynamic_data* temp_dd = dynamic_data_of (gen_idx);
22238 int added_pinning_ratio = 0;
22239 int artificial_pinned_ratio = 0;
22241 if (dd_pinned_survived_size (temp_dd) != 0)
22243 added_pinning_ratio = (int)((float)dd_added_pinned_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22244 artificial_pinned_ratio = (int)((float)dd_artificial_pinned_survived_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22247 size_t padding_size =
22249 dd_padding_size (temp_dd);
22252 #endif //SHORT_PLUGS
22253 dprintf (1, ("gen%d: %Ix, %Ix(%Id), NON PIN alloc: %Id, pin com: %Id, sweep: %Id, surv: %Id, pinsurv: %Id(%d%% added, %d%% art), np surv: %Id, pad: %Id",
22255 generation_allocation_start (temp_gen),
22256 generation_plan_allocation_start (temp_gen),
22257 (size_t)(generation_plan_allocation_start (temp_gen) - generation_allocation_start (temp_gen)),
22258 generation_allocation_size (temp_gen),
22259 generation_pinned_allocation_compact_size (temp_gen),
22260 generation_pinned_allocation_sweep_size (temp_gen),
22261 dd_survived_size (temp_dd),
22262 dd_pinned_survived_size (temp_dd),
22263 added_pinning_ratio,
22264 artificial_pinned_ratio,
22265 (dd_survived_size (temp_dd) - dd_pinned_survived_size (temp_dd)),
22268 #endif //SIMPLE_DPRINTF
22271 if (settings.condemned_generation == (max_generation - 1 ))
22273 size_t plan_gen2_size = generation_plan_size (max_generation);
22274 size_t growth = plan_gen2_size - old_gen2_size;
22278 dprintf (1, ("gen2 grew %Id (end seg alloc: %Id, gen1 c alloc: %Id",
22279 growth, generation_end_seg_allocated (generation_of (max_generation)),
22280 generation_condemned_allocated (generation_of (max_generation - 1))));
22284 dprintf (1, ("gen2 shrank %Id (end seg alloc: %Id, gen1 c alloc: %Id",
22285 (old_gen2_size - plan_gen2_size), generation_end_seg_allocated (generation_of (max_generation)),
22286 generation_condemned_allocated (generation_of (max_generation - 1))));
22289 generation* older_gen = generation_of (settings.condemned_generation + 1);
22290 size_t rejected_free_space = generation_free_obj_space (older_gen) - r_free_obj_space;
22291 size_t free_list_allocated = generation_free_list_allocated (older_gen) - r_older_gen_free_list_allocated;
22292 size_t end_seg_allocated = generation_end_seg_allocated (older_gen) - r_older_gen_end_seg_allocated;
22293 size_t condemned_allocated = generation_condemned_allocated (older_gen) - r_older_gen_condemned_allocated;
22295 dprintf (1, ("older gen's free alloc: %Id->%Id, seg alloc: %Id->%Id, condemned alloc: %Id->%Id",
22296 r_older_gen_free_list_allocated, generation_free_list_allocated (older_gen),
22297 r_older_gen_end_seg_allocated, generation_end_seg_allocated (older_gen),
22298 r_older_gen_condemned_allocated, generation_condemned_allocated (older_gen)));
22300 dprintf (1, ("this GC did %Id free list alloc(%Id bytes free space rejected), %Id seg alloc and %Id condemned alloc, gen1 condemned alloc is %Id",
22301 free_list_allocated, rejected_free_space, end_seg_allocated,
22302 condemned_allocated, generation_condemned_allocated (generation_of (settings.condemned_generation))));
22304 maxgen_size_increase* maxgen_size_info = &(get_gc_data_per_heap()->maxgen_size_info);
22305 maxgen_size_info->free_list_allocated = free_list_allocated;
22306 maxgen_size_info->free_list_rejected = rejected_free_space;
22307 maxgen_size_info->end_seg_allocated = end_seg_allocated;
22308 maxgen_size_info->condemned_allocated = condemned_allocated;
22309 maxgen_size_info->pinned_allocated = maxgen_pinned_compact_before_advance;
22310 maxgen_size_info->pinned_allocated_advance = generation_pinned_allocation_compact_size (generation_of (max_generation)) - maxgen_pinned_compact_before_advance;
22312 #ifdef FREE_USAGE_STATS
22313 int free_list_efficiency = 0;
22314 if ((free_list_allocated + rejected_free_space) != 0)
22315 free_list_efficiency = (int)(((float) (free_list_allocated) / (float)(free_list_allocated + rejected_free_space)) * (float)100);
22317 int running_free_list_efficiency = (int)(generation_allocator_efficiency(older_gen)*100);
22319 dprintf (1, ("gen%d free list alloc effi: %d%%, current effi: %d%%",
22320 older_gen->gen_num,
22321 free_list_efficiency, running_free_list_efficiency));
22323 dprintf (1, ("gen2 free list change"));
22324 for (int j = 0; j < NUM_GEN_POWER2; j++)
22326 dprintf (1, ("[h%d][#%Id]: 2^%d: F: %Id->%Id(%Id), P: %Id",
22329 (j + 10), r_older_gen_free_space[j], older_gen->gen_free_spaces[j],
22330 (ptrdiff_t)(r_older_gen_free_space[j] - older_gen->gen_free_spaces[j]),
22331 (generation_of(max_generation - 1))->gen_plugs[j]));
22333 #endif //FREE_USAGE_STATS
22336 size_t fragmentation =
22337 generation_fragmentation (generation_of (condemned_gen_number),
22339 heap_segment_allocated (ephemeral_heap_segment));
22341 dprintf (2,("Fragmentation: %Id", fragmentation));
22342 dprintf (2,("---- End of Plan phase ----"));
22345 finish = GetCycleCount32();
22346 plan_time = finish - start;
22349 // We may update write barrier code. We assume here EE has been suspended if we are on a GC thread.
22350 assert(IsGCInProgress());
22352 BOOL should_expand = FALSE;
22353 BOOL should_compact= FALSE;
22354 ephemeral_promotion = FALSE;
22357 if ((!settings.concurrent) &&
22358 ((condemned_gen_number < max_generation) &&
22359 ((settings.gen0_reduction_count > 0) || (settings.entry_memory_load >= 95))))
22361 dprintf (2, ("gen0 reduction count is %d, condemning %d, mem load %d",
22362 settings.gen0_reduction_count,
22363 condemned_gen_number,
22364 settings.entry_memory_load));
22365 should_compact = TRUE;
22367 get_gc_data_per_heap()->set_mechanism (gc_heap_compact,
22368 ((settings.gen0_reduction_count > 0) ? compact_fragmented_gen0 : compact_high_mem_load));
22370 if ((condemned_gen_number >= (max_generation - 1)) &&
22371 dt_low_ephemeral_space_p (tuning_deciding_expansion))
22373 dprintf (2, ("Not enough space for all ephemeral generations with compaction"));
22374 should_expand = TRUE;
22380 should_compact = decide_on_compacting (condemned_gen_number, fragmentation, should_expand);
22385 #ifdef FEATURE_LOH_COMPACTION
22386 loh_compacted_p = FALSE;
22387 #endif //FEATURE_LOH_COMPACTION
22389 if (condemned_gen_number == max_generation)
22391 #ifdef FEATURE_LOH_COMPACTION
22392 if (settings.loh_compaction)
22396 should_compact = TRUE;
22397 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_loh_forced);
22398 loh_compacted_p = TRUE;
22403 if ((heap_number == 0) && (loh_pinned_queue))
22405 loh_pinned_queue_decay--;
22407 if (!loh_pinned_queue_decay)
22409 delete loh_pinned_queue;
22410 loh_pinned_queue = 0;
22415 if (!loh_compacted_p)
22416 #endif //FEATURE_LOH_COMPACTION
22418 GCToEEInterface::DiagWalkLOHSurvivors(__this);
22419 sweep_large_objects();
22424 settings.loh_compaction = FALSE;
22427 #ifdef MULTIPLE_HEAPS
22429 new_heap_segment = NULL;
22431 if (should_compact && should_expand)
22432 gc_policy = policy_expand;
22433 else if (should_compact)
22434 gc_policy = policy_compact;
22436 gc_policy = policy_sweep;
22438 //vote for result of should_compact
22439 dprintf (3, ("Joining for compaction decision"));
22440 gc_t_join.join(this, gc_join_decide_on_compaction);
22441 if (gc_t_join.joined())
22443 //safe place to delete large heap segments
22444 if (condemned_gen_number == max_generation)
22446 for (int i = 0; i < n_heaps; i++)
22448 g_heaps [i]->rearrange_large_heap_segments ();
22452 settings.demotion = FALSE;
22453 int pol_max = policy_sweep;
22454 #ifdef GC_CONFIG_DRIVEN
22455 BOOL is_compaction_mandatory = FALSE;
22456 #endif //GC_CONFIG_DRIVEN
22459 for (i = 0; i < n_heaps; i++)
22461 if (pol_max < g_heaps[i]->gc_policy)
22462 pol_max = policy_compact;
22463 // set the demotion flag is any of the heap has demotion
22464 if (g_heaps[i]->demotion_high >= g_heaps[i]->demotion_low)
22466 (g_heaps[i]->get_gc_data_per_heap())->set_mechanism_bit (gc_demotion_bit);
22467 settings.demotion = TRUE;
22470 #ifdef GC_CONFIG_DRIVEN
22471 if (!is_compaction_mandatory)
22473 int compact_reason = (g_heaps[i]->get_gc_data_per_heap())->get_mechanism (gc_heap_compact);
22474 if (compact_reason >= 0)
22476 if (gc_heap_compact_reason_mandatory_p[compact_reason])
22477 is_compaction_mandatory = TRUE;
22480 #endif //GC_CONFIG_DRIVEN
22483 #ifdef GC_CONFIG_DRIVEN
22484 if (!is_compaction_mandatory)
22486 // If compaction is not mandatory we can feel free to change it to a sweeping GC.
22487 // Note that we may want to change this to only checking every so often instead of every single GC.
22488 if (should_do_sweeping_gc (pol_max >= policy_compact))
22490 pol_max = policy_sweep;
22494 if (pol_max == policy_sweep)
22495 pol_max = policy_compact;
22498 #endif //GC_CONFIG_DRIVEN
22500 for (i = 0; i < n_heaps; i++)
22502 if (pol_max > g_heaps[i]->gc_policy)
22503 g_heaps[i]->gc_policy = pol_max;
22504 //get the segment while we are serialized
22505 if (g_heaps[i]->gc_policy == policy_expand)
22507 g_heaps[i]->new_heap_segment =
22508 g_heaps[i]->soh_get_segment_to_expand();
22509 if (!g_heaps[i]->new_heap_segment)
22511 set_expand_in_full_gc (condemned_gen_number);
22512 //we are out of memory, cancel the expansion
22513 g_heaps[i]->gc_policy = policy_compact;
22518 BOOL is_full_compacting_gc = FALSE;
22520 if ((gc_policy >= policy_compact) && (condemned_gen_number == max_generation))
22522 full_gc_counts[gc_type_compacting]++;
22523 is_full_compacting_gc = TRUE;
22526 for (i = 0; i < n_heaps; i++)
22528 //copy the card and brick tables
22529 if (g_gc_card_table!= g_heaps[i]->card_table)
22531 g_heaps[i]->copy_brick_card_table();
22534 if (is_full_compacting_gc)
22536 g_heaps[i]->loh_alloc_since_cg = 0;
22540 //start all threads on the roots.
22541 dprintf(3, ("Starting all gc threads after compaction decision"));
22542 gc_t_join.restart();
22545 //reset the local variable accordingly
22546 should_compact = (gc_policy >= policy_compact);
22547 should_expand = (gc_policy >= policy_expand);
22549 #else //MULTIPLE_HEAPS
22551 //safe place to delete large heap segments
22552 if (condemned_gen_number == max_generation)
22554 rearrange_large_heap_segments ();
22557 settings.demotion = ((demotion_high >= demotion_low) ? TRUE : FALSE);
22558 if (settings.demotion)
22559 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
22561 #ifdef GC_CONFIG_DRIVEN
22562 BOOL is_compaction_mandatory = FALSE;
22563 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
22564 if (compact_reason >= 0)
22565 is_compaction_mandatory = gc_heap_compact_reason_mandatory_p[compact_reason];
22567 if (!is_compaction_mandatory)
22569 if (should_do_sweeping_gc (should_compact))
22570 should_compact = FALSE;
22572 should_compact = TRUE;
22574 #endif //GC_CONFIG_DRIVEN
22576 if (should_compact && (condemned_gen_number == max_generation))
22578 full_gc_counts[gc_type_compacting]++;
22579 loh_alloc_since_cg = 0;
22581 #endif //MULTIPLE_HEAPS
22583 if (should_compact)
22585 dprintf (2,( "**** Doing Compacting GC ****"));
22589 #ifndef MULTIPLE_HEAPS
22590 heap_segment* new_heap_segment = soh_get_segment_to_expand();
22591 #endif //!MULTIPLE_HEAPS
22592 if (new_heap_segment)
22594 consing_gen = expand_heap(condemned_gen_number,
22599 // If we couldn't get a new segment, or we were able to
22600 // reserve one but no space to commit, we couldn't
22602 if (ephemeral_heap_segment != new_heap_segment)
22604 set_expand_in_full_gc (condemned_gen_number);
22605 should_expand = FALSE;
22608 generation_allocation_limit (condemned_gen1) =
22609 generation_allocation_pointer (condemned_gen1);
22610 if ((condemned_gen_number < max_generation))
22612 generation_allocator (older_gen)->commit_alloc_list_changes();
22614 // Fix the allocation area of the older generation
22615 fix_older_allocation_area (older_gen);
22617 assert (generation_allocation_segment (consing_gen) ==
22618 ephemeral_heap_segment);
22620 GCToEEInterface::DiagWalkSurvivors(__this);
22622 relocate_phase (condemned_gen_number, first_condemned_address);
22623 compact_phase (condemned_gen_number, first_condemned_address,
22624 (!settings.demotion && settings.promotion));
22625 fix_generation_bounds (condemned_gen_number, consing_gen);
22626 assert (generation_allocation_limit (youngest_generation) ==
22627 generation_allocation_pointer (youngest_generation));
22628 if (condemned_gen_number >= (max_generation -1))
22630 #ifdef MULTIPLE_HEAPS
22631 // this needs be serialized just because we have one
22632 // segment_standby_list/seg_table for all heaps. We should make it at least
22633 // so that when hoarding is not on we don't need this join because
22634 // decommitting memory can take a long time.
22635 //must serialize on deleting segments
22636 gc_t_join.join(this, gc_join_rearrange_segs_compaction);
22637 if (gc_t_join.joined())
22639 for (int i = 0; i < n_heaps; i++)
22641 g_heaps[i]->rearrange_heap_segments(TRUE);
22643 gc_t_join.restart();
22646 rearrange_heap_segments(TRUE);
22647 #endif //MULTIPLE_HEAPS
22651 //fix the start_segment for the ephemeral generations
22652 for (int i = 0; i < max_generation; i++)
22654 generation* gen = generation_of (i);
22655 generation_start_segment (gen) = ephemeral_heap_segment;
22656 generation_allocation_segment (gen) = ephemeral_heap_segment;
22662 #ifdef FEATURE_PREMORTEM_FINALIZATION
22663 finalize_queue->UpdatePromotedGenerations (condemned_gen_number,
22664 (!settings.demotion && settings.promotion));
22665 #endif // FEATURE_PREMORTEM_FINALIZATION
22667 #ifdef MULTIPLE_HEAPS
22668 dprintf(3, ("Joining after end of compaction"));
22669 gc_t_join.join(this, gc_join_adjust_handle_age_compact);
22670 if (gc_t_join.joined())
22671 #endif //MULTIPLE_HEAPS
22673 #ifdef MULTIPLE_HEAPS
22674 //join all threads to make sure they are synchronized
22675 dprintf(3, ("Restarting after Promotion granted"));
22676 gc_t_join.restart();
22677 #endif //MULTIPLE_HEAPS
22681 sc.thread_number = heap_number;
22682 sc.promotion = FALSE;
22683 sc.concurrent = FALSE;
22684 // new generations bounds are set can call this guy
22685 if (settings.promotion && !settings.demotion)
22687 dprintf (2, ("Promoting EE roots for gen %d",
22688 condemned_gen_number));
22689 GCScan::GcPromotionsGranted(condemned_gen_number,
22690 max_generation, &sc);
22692 else if (settings.demotion)
22694 dprintf (2, ("Demoting EE roots for gen %d",
22695 condemned_gen_number));
22696 GCScan::GcDemote (condemned_gen_number, max_generation, &sc);
22701 gen0_big_free_spaces = 0;
22703 reset_pinned_queue_bos();
22704 unsigned int gen_number = min (max_generation, 1 + condemned_gen_number);
22705 generation* gen = generation_of (gen_number);
22706 uint8_t* low = generation_allocation_start (generation_of (gen_number-1));
22707 uint8_t* high = heap_segment_allocated (ephemeral_heap_segment);
22709 while (!pinned_plug_que_empty_p())
22711 mark* m = pinned_plug_of (deque_pinned_plug());
22712 size_t len = pinned_len (m);
22713 uint8_t* arr = (pinned_plug (m) - len);
22714 dprintf(3,("free [%Ix %Ix[ pin",
22715 (size_t)arr, (size_t)arr + len));
22718 assert (len >= Align (min_obj_size));
22719 make_unused_array (arr, len);
22720 // fix fully contained bricks + first one
22721 // if the array goes beyond the first brick
22722 size_t start_brick = brick_of (arr);
22723 size_t end_brick = brick_of (arr + len);
22724 if (end_brick != start_brick)
22727 ("Fixing bricks [%Ix, %Ix[ to point to unused array %Ix",
22728 start_brick, end_brick, (size_t)arr));
22729 set_brick (start_brick,
22730 arr - brick_address (start_brick));
22731 size_t brick = start_brick+1;
22732 while (brick < end_brick)
22734 set_brick (brick, start_brick - brick);
22739 //when we take an old segment to make the new
22740 //ephemeral segment. we can have a bunch of
22741 //pinned plugs out of order going to the new ephemeral seg
22742 //and then the next plugs go back to max_generation
22743 if ((heap_segment_mem (ephemeral_heap_segment) <= arr) &&
22744 (heap_segment_reserved (ephemeral_heap_segment) > arr))
22747 while ((low <= arr) && (high > arr))
22750 assert ((gen_number >= 1) || (demotion_low != MAX_PTR) ||
22751 settings.demotion || !settings.promotion);
22752 dprintf (3, ("new free list generation %d", gen_number));
22754 gen = generation_of (gen_number);
22755 if (gen_number >= 1)
22756 low = generation_allocation_start (generation_of (gen_number-1));
22763 dprintf (3, ("new free list generation %d", max_generation));
22764 gen_number = max_generation;
22765 gen = generation_of (gen_number);
22768 dprintf(3,("threading it into generation %d", gen_number));
22769 thread_gap (arr, len, gen);
22770 add_gen_free (gen_number, len);
22776 for (int x = 0; x <= max_generation; x++)
22778 assert (generation_allocation_start (generation_of (x)));
22782 if (!settings.demotion && settings.promotion)
22784 //clear card for generation 1. generation 0 is empty
22785 clear_card_for_addresses (
22786 generation_allocation_start (generation_of (1)),
22787 generation_allocation_start (generation_of (0)));
22789 if (settings.promotion && !settings.demotion)
22791 uint8_t* start = generation_allocation_start (youngest_generation);
22792 MAYBE_UNUSED_VAR(start);
22793 assert (heap_segment_allocated (ephemeral_heap_segment) ==
22794 (start + Align (size (start))));
22799 //force promotion for sweep
22800 settings.promotion = TRUE;
22801 settings.compaction = FALSE;
22804 sc.thread_number = heap_number;
22805 sc.promotion = FALSE;
22806 sc.concurrent = FALSE;
22808 dprintf (2, ("**** Doing Mark and Sweep GC****"));
22810 if ((condemned_gen_number < max_generation))
22812 generation_allocator (older_gen)->copy_from_alloc_list (r_free_list);
22813 generation_free_list_space (older_gen) = r_free_list_space;
22814 generation_free_obj_space (older_gen) = r_free_obj_space;
22815 generation_free_list_allocated (older_gen) = r_older_gen_free_list_allocated;
22816 generation_end_seg_allocated (older_gen) = r_older_gen_end_seg_allocated;
22817 generation_condemned_allocated (older_gen) = r_older_gen_condemned_allocated;
22818 generation_allocation_limit (older_gen) = r_allocation_limit;
22819 generation_allocation_pointer (older_gen) = r_allocation_pointer;
22820 generation_allocation_context_start_region (older_gen) = r_allocation_start_region;
22821 generation_allocation_segment (older_gen) = r_allocation_segment;
22824 if ((condemned_gen_number < max_generation))
22826 // Fix the allocation area of the older generation
22827 fix_older_allocation_area (older_gen);
22830 GCToEEInterface::DiagWalkSurvivors(__this);
22832 gen0_big_free_spaces = 0;
22833 make_free_lists (condemned_gen_number);
22834 recover_saved_pinned_info();
22836 #ifdef FEATURE_PREMORTEM_FINALIZATION
22837 finalize_queue->UpdatePromotedGenerations (condemned_gen_number, TRUE);
22838 #endif // FEATURE_PREMORTEM_FINALIZATION
22839 // MTHTS: leave single thread for HT processing on plan_phase
22840 #ifdef MULTIPLE_HEAPS
22841 dprintf(3, ("Joining after end of sweep"));
22842 gc_t_join.join(this, gc_join_adjust_handle_age_sweep);
22843 if (gc_t_join.joined())
22844 #endif //MULTIPLE_HEAPS
22846 GCScan::GcPromotionsGranted(condemned_gen_number,
22847 max_generation, &sc);
22848 if (condemned_gen_number >= (max_generation -1))
22850 #ifdef MULTIPLE_HEAPS
22851 for (int i = 0; i < n_heaps; i++)
22853 g_heaps[i]->rearrange_heap_segments(FALSE);
22856 rearrange_heap_segments(FALSE);
22857 #endif //MULTIPLE_HEAPS
22860 #ifdef MULTIPLE_HEAPS
22861 //join all threads to make sure they are synchronized
22862 dprintf(3, ("Restarting after Promotion granted"));
22863 gc_t_join.restart();
22864 #endif //MULTIPLE_HEAPS
22868 for (int x = 0; x <= max_generation; x++)
22870 assert (generation_allocation_start (generation_of (x)));
22874 //clear card for generation 1. generation 0 is empty
22875 clear_card_for_addresses (
22876 generation_allocation_start (generation_of (1)),
22877 generation_allocation_start (generation_of (0)));
22878 assert ((heap_segment_allocated (ephemeral_heap_segment) ==
22879 (generation_allocation_start (youngest_generation) +
22880 Align (min_obj_size))));
22883 //verify_partial();
22886 #pragma warning(pop)
22890 /*****************************
22891 Called after compact phase to fix all generation gaps
22892 ********************************/
22893 void gc_heap::fix_generation_bounds (int condemned_gen_number,
22894 generation* consing_gen)
22896 UNREFERENCED_PARAMETER(consing_gen);
22898 assert (generation_allocation_segment (consing_gen) ==
22899 ephemeral_heap_segment);
22901 //assign the planned allocation start to the generation
22902 int gen_number = condemned_gen_number;
22903 int bottom_gen = 0;
22905 while (gen_number >= bottom_gen)
22907 generation* gen = generation_of (gen_number);
22908 dprintf(3,("Fixing generation pointers for %Ix", gen_number));
22909 if ((gen_number < max_generation) && ephemeral_promotion)
22911 make_unused_array (saved_ephemeral_plan_start[gen_number],
22912 saved_ephemeral_plan_start_size[gen_number]);
22914 reset_allocation_pointers (gen, generation_plan_allocation_start (gen));
22915 make_unused_array (generation_allocation_start (gen), generation_plan_allocation_start_size (gen));
22916 dprintf(3,(" start %Ix", (size_t)generation_allocation_start (gen)));
22919 #ifdef MULTIPLE_HEAPS
22920 if (ephemeral_promotion)
22922 //we are creating a generation fault. set the cards.
22923 // and we are only doing this for multiple heaps because in the single heap scenario the
22924 // new ephemeral generations will be empty and there'll be no need to set cards for the
22925 // old ephemeral generations that got promoted into max_generation.
22926 ptrdiff_t delta = 0;
22927 #ifdef SEG_MAPPING_TABLE
22928 heap_segment* old_ephemeral_seg = seg_mapping_table_segment_of (saved_ephemeral_plan_start[max_generation-1]);
22929 #else //SEG_MAPPING_TABLE
22930 heap_segment* old_ephemeral_seg = segment_of (saved_ephemeral_plan_start[max_generation-1], delta);
22931 #endif //SEG_MAPPING_TABLE
22933 assert (in_range_for_segment (saved_ephemeral_plan_start[max_generation-1], old_ephemeral_seg));
22934 size_t end_card = card_of (align_on_card (heap_segment_plan_allocated (old_ephemeral_seg)));
22935 size_t card = card_of (saved_ephemeral_plan_start[max_generation-1]);
22936 while (card != end_card)
22942 #endif //MULTIPLE_HEAPS
22944 alloc_allocated = heap_segment_plan_allocated(ephemeral_heap_segment);
22945 //reset the allocated size
22946 uint8_t* start = generation_allocation_start (youngest_generation);
22947 MAYBE_UNUSED_VAR(start);
22948 if (settings.promotion && !settings.demotion)
22950 assert ((start + Align (size (start))) ==
22951 heap_segment_plan_allocated(ephemeral_heap_segment));
22954 heap_segment_allocated(ephemeral_heap_segment)=
22955 heap_segment_plan_allocated(ephemeral_heap_segment);
22959 uint8_t* gc_heap::generation_limit (int gen_number)
22961 if (settings.promotion)
22963 if (gen_number <= 1)
22964 return heap_segment_reserved (ephemeral_heap_segment);
22966 return generation_allocation_start (generation_of ((gen_number - 2)));
22970 if (gen_number <= 0)
22971 return heap_segment_reserved (ephemeral_heap_segment);
22973 return generation_allocation_start (generation_of ((gen_number - 1)));
22977 BOOL gc_heap::ensure_gap_allocation (int condemned_gen_number)
22979 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
22980 size_t size = Align (min_obj_size)*(condemned_gen_number+1);
22981 assert ((start + size) <=
22982 heap_segment_reserved (ephemeral_heap_segment));
22983 if ((start + size) >
22984 heap_segment_committed (ephemeral_heap_segment))
22986 if (!grow_heap_segment (ephemeral_heap_segment, start + size))
22994 uint8_t* gc_heap::allocate_at_end (size_t size)
22996 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
22997 size = Align (size);
22998 uint8_t* result = start;
22999 // only called to allocate a min obj so can't overflow here.
23000 assert ((start + size) <=
23001 heap_segment_reserved (ephemeral_heap_segment));
23002 //ensure_gap_allocation took care of it
23003 assert ((start + size) <=
23004 heap_segment_committed (ephemeral_heap_segment));
23005 heap_segment_allocated (ephemeral_heap_segment) += size;
23010 void gc_heap::make_free_lists (int condemned_gen_number)
23015 start = GetCycleCount32();
23018 //Promotion has to happen in sweep case.
23019 assert (settings.promotion);
23021 generation* condemned_gen = generation_of (condemned_gen_number);
23022 uint8_t* start_address = generation_allocation_start (condemned_gen);
23024 size_t current_brick = brick_of (start_address);
23025 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
23027 PREFIX_ASSUME(current_heap_segment != NULL);
23029 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
23030 size_t end_brick = brick_of (end_address-1);
23031 make_free_args args;
23032 args.free_list_gen_number = min (max_generation, 1 + condemned_gen_number);
23033 args.current_gen_limit = (((condemned_gen_number == max_generation)) ?
23035 (generation_limit (args.free_list_gen_number)));
23036 args.free_list_gen = generation_of (args.free_list_gen_number);
23037 args.highest_plug = 0;
23039 if ((start_address < end_address) ||
23040 (condemned_gen_number == max_generation))
23044 if ((current_brick > end_brick))
23046 if (args.current_gen_limit == MAX_PTR)
23048 //We had an empty segment
23049 //need to allocate the generation start
23051 generation* gen = generation_of (max_generation);
23053 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
23055 PREFIX_ASSUME(start_seg != NULL);
23057 uint8_t* gap = heap_segment_mem (start_seg);
23059 generation_allocation_start (gen) = gap;
23060 heap_segment_allocated (start_seg) = gap + Align (min_obj_size);
23061 make_unused_array (gap, Align (min_obj_size));
23062 reset_allocation_pointers (gen, gap);
23063 dprintf (3, ("Start segment empty, fixing generation start of %d to: %Ix",
23064 max_generation, (size_t)gap));
23065 args.current_gen_limit = generation_limit (args.free_list_gen_number);
23067 if (heap_segment_next_rw (current_heap_segment))
23069 current_heap_segment = heap_segment_next_rw (current_heap_segment);
23070 current_brick = brick_of (heap_segment_mem (current_heap_segment));
23071 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
23081 int brick_entry = brick_table [ current_brick ];
23082 if ((brick_entry >= 0))
23084 make_free_list_in_brick (brick_address (current_brick) + brick_entry-1, &args);
23085 dprintf(3,("Fixing brick entry %Ix to %Ix",
23086 current_brick, (size_t)args.highest_plug));
23087 set_brick (current_brick,
23088 (args.highest_plug - brick_address (current_brick)));
23092 if ((brick_entry > -32768))
23096 ptrdiff_t offset = brick_of (args.highest_plug) - current_brick;
23097 if ((brick_entry != -32767) && (! ((offset == brick_entry))))
23099 assert ((brick_entry == -1));
23102 //init to -1 for faster find_first_object
23103 set_brick (current_brick, -1);
23111 int bottom_gen = 0;
23112 args.free_list_gen_number--;
23113 while (args.free_list_gen_number >= bottom_gen)
23116 generation* gen2 = generation_of (args.free_list_gen_number);
23117 gap = allocate_at_end (Align(min_obj_size));
23118 generation_allocation_start (gen2) = gap;
23119 reset_allocation_pointers (gen2, gap);
23120 dprintf(3,("Fixing generation start of %d to: %Ix",
23121 args.free_list_gen_number, (size_t)gap));
23122 PREFIX_ASSUME(gap != NULL);
23123 make_unused_array (gap, Align (min_obj_size));
23125 args.free_list_gen_number--;
23128 //reset the allocated size
23129 uint8_t* start2 = generation_allocation_start (youngest_generation);
23130 alloc_allocated = start2 + Align (size (start2));
23134 finish = GetCycleCount32();
23135 sweep_time = finish - start;
23139 void gc_heap::make_free_list_in_brick (uint8_t* tree, make_free_args* args)
23141 assert ((tree != NULL));
23143 int right_node = node_right_child (tree);
23144 int left_node = node_left_child (tree);
23145 args->highest_plug = 0;
23148 if (! (0 == left_node))
23150 make_free_list_in_brick (tree + left_node, args);
23154 uint8_t* plug = tree;
23155 size_t gap_size = node_gap_size (tree);
23156 uint8_t* gap = (plug - gap_size);
23157 dprintf (3,("Making free list %Ix len %d in %d",
23158 //dprintf (3,("F: %Ix len %Ix in %d",
23159 (size_t)gap, gap_size, args->free_list_gen_number));
23160 args->highest_plug = tree;
23162 if (is_plug_padded (plug))
23164 dprintf (3, ("%Ix padded", plug));
23165 clear_plug_padded (plug);
23167 #endif //SHORT_PLUGS
23170 if ((args->current_gen_limit == MAX_PTR) ||
23171 ((plug >= args->current_gen_limit) &&
23172 ephemeral_pointer_p (plug)))
23174 dprintf(3,(" Crossing Generation boundary at %Ix",
23175 (size_t)args->current_gen_limit));
23176 if (!(args->current_gen_limit == MAX_PTR))
23178 args->free_list_gen_number--;
23179 args->free_list_gen = generation_of (args->free_list_gen_number);
23181 dprintf(3,( " Fixing generation start of %d to: %Ix",
23182 args->free_list_gen_number, (size_t)gap));
23184 reset_allocation_pointers (args->free_list_gen, gap);
23185 args->current_gen_limit = generation_limit (args->free_list_gen_number);
23187 if ((gap_size >= (2*Align (min_obj_size))))
23189 dprintf(3,(" Splitting the gap in two %Id left",
23191 make_unused_array (gap, Align(min_obj_size));
23192 gap_size = (gap_size - Align(min_obj_size));
23193 gap = (gap + Align(min_obj_size));
23197 make_unused_array (gap, gap_size);
23204 thread_gap (gap, gap_size, args->free_list_gen);
23205 add_gen_free (args->free_list_gen->gen_num, gap_size);
23207 if (! (0 == right_node))
23209 make_free_list_in_brick (tree + right_node, args);
23215 void gc_heap::thread_gap (uint8_t* gap_start, size_t size, generation* gen)
23217 assert (generation_allocation_start (gen));
23220 if ((gen->gen_num == 0) && (size > CLR_SIZE))
23222 gen0_big_free_spaces += size;
23225 assert ((heap_segment_rw (generation_start_segment (gen))!=
23226 ephemeral_heap_segment) ||
23227 (gap_start > generation_allocation_start (gen)));
23228 // The beginning of a segment gap is not aligned
23229 assert (size >= Align (min_obj_size));
23230 make_unused_array (gap_start, size,
23231 (!settings.concurrent && (gen != youngest_generation)),
23232 (gen->gen_num == max_generation));
23233 dprintf (3, ("fr: [%Ix, %Ix[", (size_t)gap_start, (size_t)gap_start+size));
23235 if ((size >= min_free_list))
23237 generation_free_list_space (gen) += size;
23238 generation_allocator (gen)->thread_item (gap_start, size);
23242 generation_free_obj_space (gen) += size;
23247 void gc_heap::loh_thread_gap_front (uint8_t* gap_start, size_t size, generation* gen)
23249 assert (generation_allocation_start (gen));
23250 if (size >= min_free_list)
23252 generation_free_list_space (gen) += size;
23253 generation_allocator (gen)->thread_item_front (gap_start, size);
23257 void gc_heap::make_unused_array (uint8_t* x, size_t size, BOOL clearp, BOOL resetp)
23259 dprintf (3, ("Making unused array [%Ix, %Ix[",
23260 (size_t)x, (size_t)(x+size)));
23261 assert (size >= Align (min_obj_size));
23263 //#if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
23264 // check_batch_mark_array_bits (x, x+size);
23265 //#endif //VERIFY_HEAP && BACKGROUND_GC
23268 reset_memory (x, size);
23270 ((CObjectHeader*)x)->SetFree(size);
23275 #error "This won't work on big endian platforms"
23278 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23280 if (size_as_object < size)
23283 // If the size is more than 4GB, we need to create multiple objects because of
23284 // the Array::m_NumComponents is uint32_t and the high 32 bits of unused array
23285 // size is ignored in regular object size computation.
23287 uint8_t * tmp = x + size_as_object;
23288 size_t remaining_size = size - size_as_object;
23290 while (remaining_size > UINT32_MAX)
23292 // Make sure that there will be at least Align(min_obj_size) left
23293 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23294 - Align (min_obj_size, get_alignment_constant (FALSE));
23296 ((CObjectHeader*)tmp)->SetFree(current_size);
23298 remaining_size -= current_size;
23299 tmp += current_size;
23302 ((CObjectHeader*)tmp)->SetFree(remaining_size);
23307 clear_card_for_addresses (x, x + Align(size));
23310 // Clear memory set by make_unused_array.
23311 void gc_heap::clear_unused_array (uint8_t* x, size_t size)
23313 // Also clear the sync block
23314 *(((PTR_PTR)x)-1) = 0;
23316 ((CObjectHeader*)x)->UnsetFree();
23321 #error "This won't work on big endian platforms"
23324 // The memory could have been cleared in the meantime. We have to mirror the algorithm
23325 // from make_unused_array since we cannot depend on the object sizes in memory.
23326 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23328 if (size_as_object < size)
23330 uint8_t * tmp = x + size_as_object;
23331 size_t remaining_size = size - size_as_object;
23333 while (remaining_size > UINT32_MAX)
23335 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23336 - Align (min_obj_size, get_alignment_constant (FALSE));
23338 ((CObjectHeader*)tmp)->UnsetFree();
23340 remaining_size -= current_size;
23341 tmp += current_size;
23344 ((CObjectHeader*)tmp)->UnsetFree();
23347 UNREFERENCED_PARAMETER(size);
23352 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address)
23354 uint8_t* candidate = 0;
23358 if (tree < old_address)
23360 if ((cn = node_right_child (tree)) != 0)
23362 assert (candidate < tree);
23365 Prefetch (tree - 8);
23371 else if (tree > old_address)
23373 if ((cn = node_left_child (tree)) != 0)
23376 Prefetch (tree - 8);
23384 if (tree <= old_address)
23386 else if (candidate)
23392 #ifdef FEATURE_BASICFREEZE
23393 bool gc_heap::frozen_object_p (Object* obj)
23395 heap_segment* pSegment = gc_heap::find_segment ((uint8_t*)obj, FALSE);
23396 _ASSERTE(pSegment);
23398 return heap_segment_read_only_p(pSegment);
23400 #endif // FEATURE_BASICFREEZE
23402 #ifdef FEATURE_REDHAWK
23403 // TODO: this was added on RH, we have not done perf runs to see if this is the right
23404 // thing to do for other versions of the CLR.
23406 #endif // FEATURE_REDHAWK
23407 void gc_heap::relocate_address (uint8_t** pold_address THREAD_NUMBER_DCL)
23409 uint8_t* old_address = *pold_address;
23410 if (!((old_address >= gc_low) && (old_address < gc_high)))
23411 #ifdef MULTIPLE_HEAPS
23413 UNREFERENCED_PARAMETER(thread);
23414 if (old_address == 0)
23416 gc_heap* hp = heap_of (old_address);
23417 if ((hp == this) ||
23418 !((old_address >= hp->gc_low) && (old_address < hp->gc_high)))
23421 #else //MULTIPLE_HEAPS
23423 #endif //MULTIPLE_HEAPS
23424 // delta translates old_address into address_gc (old_address);
23425 size_t brick = brick_of (old_address);
23426 int brick_entry = brick_table [ brick ];
23427 uint8_t* new_address = old_address;
23428 if (! ((brick_entry == 0)))
23432 while (brick_entry < 0)
23434 brick = (brick + brick_entry);
23435 brick_entry = brick_table [ brick ];
23437 uint8_t* old_loc = old_address;
23439 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
23441 if ((node <= old_loc))
23442 new_address = (old_address + node_relocation_distance (node));
23445 if (node_left_p (node))
23447 dprintf(3,(" L: %Ix", (size_t)node));
23448 new_address = (old_address +
23449 (node_relocation_distance (node) +
23450 node_gap_size (node)));
23455 brick_entry = brick_table [ brick ];
23461 *pold_address = new_address;
23465 #ifdef FEATURE_LOH_COMPACTION
23466 if (loh_compacted_p
23467 #ifdef FEATURE_BASICFREEZE
23468 && !frozen_object_p((Object*)old_address)
23469 #endif // FEATURE_BASICFREEZE
23472 *pold_address = old_address + loh_node_relocation_distance (old_address);
23475 #endif //FEATURE_LOH_COMPACTION
23477 *pold_address = new_address;
23482 gc_heap::check_class_object_demotion (uint8_t* obj)
23484 #ifdef COLLECTIBLE_CLASS
23485 if (is_collectible(obj))
23487 check_class_object_demotion_internal (obj);
23490 UNREFERENCED_PARAMETER(obj);
23491 #endif //COLLECTIBLE_CLASS
23494 #ifdef COLLECTIBLE_CLASS
23496 gc_heap::check_class_object_demotion_internal (uint8_t* obj)
23498 if (settings.demotion)
23500 #ifdef MULTIPLE_HEAPS
23501 // We set the card without checking the demotion range 'cause at this point
23502 // the handle that points to the loader allocator object may or may not have
23503 // been relocated by other GC threads.
23504 set_card (card_of (obj));
23507 uint8_t* class_obj = get_class_object (obj);
23508 dprintf (3, ("%Ix: got classobj %Ix", obj, class_obj));
23509 uint8_t* temp_class_obj = class_obj;
23510 uint8_t** temp = &temp_class_obj;
23511 relocate_address (temp THREAD_NUMBER_ARG);
23513 check_demotion_helper (temp, obj);
23514 #endif //MULTIPLE_HEAPS
23518 #endif //COLLECTIBLE_CLASS
23521 gc_heap::check_demotion_helper (uint8_t** pval, uint8_t* parent_obj)
23523 // detect if we are demoting an object
23524 if ((*pval < demotion_high) &&
23525 (*pval >= demotion_low))
23527 dprintf(3, ("setting card %Ix:%Ix",
23528 card_of((uint8_t*)pval),
23531 set_card (card_of (parent_obj));
23533 #ifdef MULTIPLE_HEAPS
23534 else if (settings.demotion)
23536 dprintf (4, ("Demotion active, computing heap_of object"));
23537 gc_heap* hp = heap_of (*pval);
23538 if ((*pval < hp->demotion_high) &&
23539 (*pval >= hp->demotion_low))
23541 dprintf(3, ("setting card %Ix:%Ix",
23542 card_of((uint8_t*)pval),
23545 set_card (card_of (parent_obj));
23548 #endif //MULTIPLE_HEAPS
23552 gc_heap::reloc_survivor_helper (uint8_t** pval)
23555 relocate_address (pval THREAD_NUMBER_ARG);
23557 check_demotion_helper (pval, (uint8_t*)pval);
23561 gc_heap::relocate_obj_helper (uint8_t* x, size_t s)
23564 if (contain_pointers (x))
23566 dprintf (3, ("$%Ix$", (size_t)x));
23568 go_through_object_nostart (method_table(x), x, s, pval,
23570 uint8_t* child = *pval;
23571 reloc_survivor_helper (pval);
23574 dprintf (3, ("%Ix->%Ix->%Ix", (uint8_t*)pval, child, *pval));
23579 check_class_object_demotion (x);
23583 void gc_heap::reloc_ref_in_shortened_obj (uint8_t** address_to_set_card, uint8_t** address_to_reloc)
23587 uint8_t* old_val = (address_to_reloc ? *address_to_reloc : 0);
23588 relocate_address (address_to_reloc THREAD_NUMBER_ARG);
23589 if (address_to_reloc)
23591 dprintf (3, ("SR %Ix: %Ix->%Ix", (uint8_t*)address_to_reloc, old_val, *address_to_reloc));
23594 //check_demotion_helper (current_saved_info_to_relocate, (uint8_t*)pval);
23595 uint8_t* relocated_addr = *address_to_reloc;
23596 if ((relocated_addr < demotion_high) &&
23597 (relocated_addr >= demotion_low))
23599 dprintf (3, ("set card for location %Ix(%Ix)",
23600 (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
23602 set_card (card_of ((uint8_t*)address_to_set_card));
23604 #ifdef MULTIPLE_HEAPS
23605 else if (settings.demotion)
23607 gc_heap* hp = heap_of (relocated_addr);
23608 if ((relocated_addr < hp->demotion_high) &&
23609 (relocated_addr >= hp->demotion_low))
23611 dprintf (3, ("%Ix on h%d, set card for location %Ix(%Ix)",
23612 relocated_addr, hp->heap_number, (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
23614 set_card (card_of ((uint8_t*)address_to_set_card));
23617 #endif //MULTIPLE_HEAPS
23620 void gc_heap::relocate_pre_plug_info (mark* pinned_plug_entry)
23623 uint8_t* plug = pinned_plug (pinned_plug_entry);
23624 uint8_t* pre_plug_start = plug - sizeof (plug_and_gap);
23625 // Note that we need to add one ptr size here otherwise we may not be able to find the relocated
23626 // address. Consider this scenario:
23627 // gen1 start | 3-ptr sized NP | PP
23629 // If we are asking for the reloc address of 0x10 we will AV in relocate_address because
23630 // the first plug we saw in the brick is 0x18 which means 0x10 will cause us to go back a brick
23631 // which is 0, and then we'll AV in tree_search when we try to do node_right_child (tree).
23632 pre_plug_start += sizeof (uint8_t*);
23633 uint8_t** old_address = &pre_plug_start;
23635 uint8_t* old_val = (old_address ? *old_address : 0);
23636 relocate_address (old_address THREAD_NUMBER_ARG);
23639 dprintf (3, ("PreR %Ix: %Ix->%Ix, set reloc: %Ix",
23640 (uint8_t*)old_address, old_val, *old_address, (pre_plug_start - sizeof (uint8_t*))));
23643 pinned_plug_entry->set_pre_plug_info_reloc_start (pre_plug_start - sizeof (uint8_t*));
23647 void gc_heap::relocate_shortened_obj_helper (uint8_t* x, size_t s, uint8_t* end, mark* pinned_plug_entry, BOOL is_pinned)
23650 uint8_t* plug = pinned_plug (pinned_plug_entry);
23654 //// Temporary - we just wanna make sure we are doing things right when padding is needed.
23655 //if ((x + s) < plug)
23657 // dprintf (3, ("obj %Ix needed padding: end %Ix is %d bytes from pinned obj %Ix",
23658 // x, (x + s), (plug- (x + s)), plug));
23659 // GCToOSInterface::DebugBreak();
23662 relocate_pre_plug_info (pinned_plug_entry);
23665 verify_pins_with_post_plug_info("after relocate_pre_plug_info");
23667 uint8_t* saved_plug_info_start = 0;
23668 uint8_t** saved_info_to_relocate = 0;
23672 saved_plug_info_start = (uint8_t*)(pinned_plug_entry->get_post_plug_info_start());
23673 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
23677 saved_plug_info_start = (plug - sizeof (plug_and_gap));
23678 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
23681 uint8_t** current_saved_info_to_relocate = 0;
23682 uint8_t* child = 0;
23684 dprintf (3, ("x: %Ix, pp: %Ix, end: %Ix", x, plug, end));
23686 if (contain_pointers (x))
23688 dprintf (3,("$%Ix$", (size_t)x));
23690 go_through_object_nostart (method_table(x), x, s, pval,
23692 dprintf (3, ("obj %Ix, member: %Ix->%Ix", x, (uint8_t*)pval, *pval));
23694 if ((uint8_t*)pval >= end)
23696 current_saved_info_to_relocate = saved_info_to_relocate + ((uint8_t*)pval - saved_plug_info_start) / sizeof (uint8_t**);
23697 child = *current_saved_info_to_relocate;
23698 reloc_ref_in_shortened_obj (pval, current_saved_info_to_relocate);
23699 dprintf (3, ("last part: R-%Ix(saved: %Ix)->%Ix ->%Ix",
23700 (uint8_t*)pval, current_saved_info_to_relocate, child, *current_saved_info_to_relocate));
23704 reloc_survivor_helper (pval);
23709 check_class_object_demotion (x);
23712 void gc_heap::relocate_survivor_helper (uint8_t* plug, uint8_t* plug_end)
23715 while (x < plug_end)
23717 size_t s = size (x);
23718 uint8_t* next_obj = x + Align (s);
23719 Prefetch (next_obj);
23720 relocate_obj_helper (x, s);
23726 // if we expanded, right now we are not handling it as We are not saving the new reloc info.
23727 void gc_heap::verify_pins_with_post_plug_info (const char* msg)
23729 #if defined (_DEBUG) && defined (VERIFY_HEAP)
23730 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
23732 if (!verify_pinned_queue_p)
23735 if (settings.heap_expansion)
23738 for (size_t i = 0; i < mark_stack_tos; i++)
23740 mark& m = mark_stack_array[i];
23742 mark* pinned_plug_entry = pinned_plug_of(i);
23744 if (pinned_plug_entry->has_post_plug_info() &&
23745 pinned_plug_entry->post_short_p() &&
23746 (pinned_plug_entry->saved_post_plug_debug.gap != 1))
23748 uint8_t* next_obj = pinned_plug_entry->get_post_plug_info_start() + sizeof (plug_and_gap);
23749 // object after pin
23750 dprintf (3, ("OFP: %Ix, G: %Ix, R: %Ix, LC: %d, RC: %d",
23751 next_obj, node_gap_size (next_obj), node_relocation_distance (next_obj),
23752 (int)node_left_child (next_obj), (int)node_right_child (next_obj)));
23754 size_t* post_plug_debug = (size_t*)(&m.saved_post_plug_debug);
23756 if (node_gap_size (next_obj) != *post_plug_debug)
23758 dprintf (3, ("obj: %Ix gap should be %Ix but it is %Ix",
23759 next_obj, *post_plug_debug, (size_t)(node_gap_size (next_obj))));
23763 // can't do node_relocation_distance here as it clears the left bit.
23764 //if (node_relocation_distance (next_obj) != *post_plug_debug)
23765 if (*((size_t*)(next_obj - 3 * sizeof (size_t))) != *post_plug_debug)
23767 dprintf (3, ("obj: %Ix reloc should be %Ix but it is %Ix",
23768 next_obj, *post_plug_debug, (size_t)(node_relocation_distance (next_obj))));
23771 if (node_left_child (next_obj) > 0)
23773 dprintf (3, ("obj: %Ix, vLC: %d\n", next_obj, (int)(node_left_child (next_obj))));
23779 dprintf (3, ("%s verified", msg));
23781 #else // _DEBUG && VERIFY_HEAP
23782 UNREFERENCED_PARAMETER(msg);
23783 #endif // _DEBUG && VERIFY_HEAP
23786 #ifdef COLLECTIBLE_CLASS
23787 // We don't want to burn another ptr size space for pinned plugs to record this so just
23788 // set the card unconditionally for collectible objects if we are demoting.
23790 gc_heap::unconditional_set_card_collectible (uint8_t* obj)
23792 if (settings.demotion)
23794 set_card (card_of (obj));
23797 #endif //COLLECTIBLE_CLASS
23799 void gc_heap::relocate_shortened_survivor_helper (uint8_t* plug, uint8_t* plug_end, mark* pinned_plug_entry)
23802 uint8_t* p_plug = pinned_plug (pinned_plug_entry);
23803 BOOL is_pinned = (plug == p_plug);
23804 BOOL check_short_obj_p = (is_pinned ? pinned_plug_entry->post_short_p() : pinned_plug_entry->pre_short_p());
23806 plug_end += sizeof (gap_reloc_pair);
23808 //dprintf (3, ("%s %Ix is shortened, and last object %s overwritten", (is_pinned ? "PP" : "NP"), plug, (check_short_obj_p ? "is" : "is not")));
23809 dprintf (3, ("%s %Ix-%Ix short, LO: %s OW", (is_pinned ? "PP" : "NP"), plug, plug_end, (check_short_obj_p ? "is" : "is not")));
23811 verify_pins_with_post_plug_info("begin reloc short surv");
23813 while (x < plug_end)
23815 if (check_short_obj_p && ((plug_end - x) < min_pre_pin_obj_size))
23817 dprintf (3, ("last obj %Ix is short", x));
23821 #ifdef COLLECTIBLE_CLASS
23822 if (pinned_plug_entry->post_short_collectible_p())
23823 unconditional_set_card_collectible (x);
23824 #endif //COLLECTIBLE_CLASS
23826 // Relocate the saved references based on bits set.
23827 uint8_t** saved_plug_info_start = (uint8_t**)(pinned_plug_entry->get_post_plug_info_start());
23828 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
23829 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
23831 if (pinned_plug_entry->post_short_bit_p (i))
23833 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
23839 #ifdef COLLECTIBLE_CLASS
23840 if (pinned_plug_entry->pre_short_collectible_p())
23841 unconditional_set_card_collectible (x);
23842 #endif //COLLECTIBLE_CLASS
23844 relocate_pre_plug_info (pinned_plug_entry);
23846 // Relocate the saved references based on bits set.
23847 uint8_t** saved_plug_info_start = (uint8_t**)(p_plug - sizeof (plug_and_gap));
23848 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
23849 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
23851 if (pinned_plug_entry->pre_short_bit_p (i))
23853 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
23861 size_t s = size (x);
23862 uint8_t* next_obj = x + Align (s);
23863 Prefetch (next_obj);
23865 if (next_obj >= plug_end)
23867 dprintf (3, ("object %Ix is at the end of the plug %Ix->%Ix",
23868 next_obj, plug, plug_end));
23870 verify_pins_with_post_plug_info("before reloc short obj");
23872 relocate_shortened_obj_helper (x, s, (x + Align (s) - sizeof (plug_and_gap)), pinned_plug_entry, is_pinned);
23876 relocate_obj_helper (x, s);
23883 verify_pins_with_post_plug_info("end reloc short surv");
23886 void gc_heap::relocate_survivors_in_plug (uint8_t* plug, uint8_t* plug_end,
23887 BOOL check_last_object_p,
23888 mark* pinned_plug_entry)
23890 //dprintf(3,("Relocating pointers in Plug [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
23891 dprintf (3,("RP: [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
23893 if (check_last_object_p)
23895 relocate_shortened_survivor_helper (plug, plug_end, pinned_plug_entry);
23899 relocate_survivor_helper (plug, plug_end);
23903 void gc_heap::relocate_survivors_in_brick (uint8_t* tree, relocate_args* args)
23905 assert ((tree != NULL));
23907 dprintf (3, ("tree: %Ix, args->last_plug: %Ix, left: %Ix, right: %Ix, gap(t): %Ix",
23908 tree, args->last_plug,
23909 (tree + node_left_child (tree)),
23910 (tree + node_right_child (tree)),
23911 node_gap_size (tree)));
23913 if (node_left_child (tree))
23915 relocate_survivors_in_brick (tree + node_left_child (tree), args);
23918 uint8_t* plug = tree;
23919 BOOL has_post_plug_info_p = FALSE;
23920 BOOL has_pre_plug_info_p = FALSE;
23922 if (tree == oldest_pinned_plug)
23924 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
23925 &has_post_plug_info_p);
23926 assert (tree == pinned_plug (args->pinned_plug_entry));
23928 dprintf (3, ("tree is the oldest pin: %Ix", tree));
23930 if (args->last_plug)
23932 size_t gap_size = node_gap_size (tree);
23933 uint8_t* gap = (plug - gap_size);
23934 dprintf (3, ("tree: %Ix, gap: %Ix (%Ix)", tree, gap, gap_size));
23935 assert (gap_size >= Align (min_obj_size));
23936 uint8_t* last_plug_end = gap;
23938 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
23941 relocate_survivors_in_plug (args->last_plug, last_plug_end, check_last_object_p, args->pinned_plug_entry);
23946 assert (!has_pre_plug_info_p);
23949 args->last_plug = plug;
23950 args->is_shortened = has_post_plug_info_p;
23951 if (has_post_plug_info_p)
23953 dprintf (3, ("setting %Ix as shortened", plug));
23955 dprintf (3, ("last_plug: %Ix(shortened: %d)", plug, (args->is_shortened ? 1 : 0)));
23957 if (node_right_child (tree))
23959 relocate_survivors_in_brick (tree + node_right_child (tree), args);
23964 void gc_heap::update_oldest_pinned_plug()
23966 oldest_pinned_plug = (pinned_plug_que_empty_p() ? 0 : pinned_plug (oldest_pin()));
23969 void gc_heap::relocate_survivors (int condemned_gen_number,
23970 uint8_t* first_condemned_address)
23972 generation* condemned_gen = generation_of (condemned_gen_number);
23973 uint8_t* start_address = first_condemned_address;
23974 size_t current_brick = brick_of (start_address);
23975 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
23977 PREFIX_ASSUME(current_heap_segment != NULL);
23979 uint8_t* end_address = 0;
23981 reset_pinned_queue_bos();
23982 update_oldest_pinned_plug();
23984 end_address = heap_segment_allocated (current_heap_segment);
23986 size_t end_brick = brick_of (end_address - 1);
23987 relocate_args args;
23989 args.high = gc_high;
23990 args.is_shortened = FALSE;
23991 args.pinned_plug_entry = 0;
23992 args.last_plug = 0;
23995 if (current_brick > end_brick)
23997 if (args.last_plug)
24000 assert (!(args.is_shortened));
24001 relocate_survivors_in_plug (args.last_plug,
24002 heap_segment_allocated (current_heap_segment),
24004 args.pinned_plug_entry);
24007 args.last_plug = 0;
24010 if (heap_segment_next_rw (current_heap_segment))
24012 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24013 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24014 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24023 int brick_entry = brick_table [ current_brick ];
24025 if (brick_entry >= 0)
24027 relocate_survivors_in_brick (brick_address (current_brick) +
24036 void gc_heap::walk_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, walk_relocate_args* args)
24038 if (check_last_object_p)
24040 size += sizeof (gap_reloc_pair);
24041 mark* entry = args->pinned_plug_entry;
24043 if (args->is_shortened)
24045 assert (entry->has_post_plug_info());
24046 entry->swap_post_plug_and_saved_for_profiler();
24050 assert (entry->has_pre_plug_info());
24051 entry->swap_pre_plug_and_saved_for_profiler();
24055 ptrdiff_t last_plug_relocation = node_relocation_distance (plug);
24056 STRESS_LOG_PLUG_MOVE(plug, (plug + size), -last_plug_relocation);
24057 ptrdiff_t reloc = settings.compaction ? last_plug_relocation : 0;
24059 (args->fn) (plug, (plug + size), reloc, args->profiling_context, !!settings.compaction, false);
24061 if (check_last_object_p)
24063 mark* entry = args->pinned_plug_entry;
24065 if (args->is_shortened)
24067 entry->swap_post_plug_and_saved_for_profiler();
24071 entry->swap_pre_plug_and_saved_for_profiler();
24076 void gc_heap::walk_relocation_in_brick (uint8_t* tree, walk_relocate_args* args)
24078 assert ((tree != NULL));
24079 if (node_left_child (tree))
24081 walk_relocation_in_brick (tree + node_left_child (tree), args);
24084 uint8_t* plug = tree;
24085 BOOL has_pre_plug_info_p = FALSE;
24086 BOOL has_post_plug_info_p = FALSE;
24088 if (tree == oldest_pinned_plug)
24090 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24091 &has_post_plug_info_p);
24092 assert (tree == pinned_plug (args->pinned_plug_entry));
24095 if (args->last_plug != 0)
24097 size_t gap_size = node_gap_size (tree);
24098 uint8_t* gap = (plug - gap_size);
24099 uint8_t* last_plug_end = gap;
24100 size_t last_plug_size = (last_plug_end - args->last_plug);
24101 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
24102 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
24104 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24105 if (!check_last_object_p)
24107 assert (last_plug_size >= Align (min_obj_size));
24110 walk_plug (args->last_plug, last_plug_size, check_last_object_p, args);
24114 assert (!has_pre_plug_info_p);
24117 dprintf (3, ("set args last plug to plug: %Ix", plug));
24118 args->last_plug = plug;
24119 args->is_shortened = has_post_plug_info_p;
24121 if (node_right_child (tree))
24123 walk_relocation_in_brick (tree + node_right_child (tree), args);
24127 void gc_heap::walk_relocation (void* profiling_context, record_surv_fn fn)
24129 generation* condemned_gen = generation_of (settings.condemned_generation);
24130 uint8_t* start_address = generation_allocation_start (condemned_gen);
24131 size_t current_brick = brick_of (start_address);
24132 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24134 PREFIX_ASSUME(current_heap_segment != NULL);
24136 reset_pinned_queue_bos();
24137 update_oldest_pinned_plug();
24138 size_t end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24139 walk_relocate_args args;
24140 args.is_shortened = FALSE;
24141 args.pinned_plug_entry = 0;
24142 args.last_plug = 0;
24143 args.profiling_context = profiling_context;
24148 if (current_brick > end_brick)
24150 if (args.last_plug)
24152 walk_plug (args.last_plug,
24153 (heap_segment_allocated (current_heap_segment) - args.last_plug),
24156 args.last_plug = 0;
24158 if (heap_segment_next_rw (current_heap_segment))
24160 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24161 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24162 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24171 int brick_entry = brick_table [ current_brick ];
24172 if (brick_entry >= 0)
24174 walk_relocation_in_brick (brick_address (current_brick) +
24183 void gc_heap::walk_survivors (record_surv_fn fn, void* context, walk_surv_type type)
24185 if (type == walk_for_gc)
24186 walk_survivors_relocation (context, fn);
24187 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24188 else if (type == walk_for_bgc)
24189 walk_survivors_for_bgc (context, fn);
24190 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
24191 else if (type == walk_for_loh)
24192 walk_survivors_for_loh (context, fn);
24194 assert (!"unknown type!");
24197 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24198 void gc_heap::walk_survivors_for_bgc (void* profiling_context, record_surv_fn fn)
24200 // This should only be called for BGCs
24201 assert(settings.concurrent);
24203 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
24205 BOOL small_object_segments = TRUE;
24206 int align_const = get_alignment_constant (small_object_segments);
24212 if (small_object_segments)
24214 //switch to large segment
24215 small_object_segments = FALSE;
24217 align_const = get_alignment_constant (small_object_segments);
24218 seg = heap_segment_rw (generation_start_segment (large_object_generation));
24220 PREFIX_ASSUME(seg != NULL);
24228 uint8_t* o = heap_segment_mem (seg);
24229 uint8_t* end = heap_segment_allocated (seg);
24233 if (method_table(o) == g_gc_pFreeObjectMethodTable)
24235 o += Align (size (o), align_const);
24239 // It's survived. Make a fake plug, starting at o,
24240 // and send the event
24242 uint8_t* plug_start = o;
24244 while (method_table(o) != g_gc_pFreeObjectMethodTable)
24246 o += Align (size (o), align_const);
24253 uint8_t* plug_end = o;
24257 0, // Reloc distance == 0 as this is non-compacting
24259 false, // Non-compacting
24263 seg = heap_segment_next (seg);
24266 #endif // defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24268 void gc_heap::relocate_phase (int condemned_gen_number,
24269 uint8_t* first_condemned_address)
24272 sc.thread_number = heap_number;
24273 sc.promotion = FALSE;
24274 sc.concurrent = FALSE;
24280 start = GetCycleCount32();
24283 // %type% category = quote (relocate);
24284 dprintf (2,("---- Relocate phase -----"));
24286 #ifdef MULTIPLE_HEAPS
24287 //join all threads to make sure they are synchronized
24288 dprintf(3, ("Joining after end of plan"));
24289 gc_t_join.join(this, gc_join_begin_relocate_phase);
24290 if (gc_t_join.joined())
24291 #endif //MULTIPLE_HEAPS
24294 #ifdef MULTIPLE_HEAPS
24296 //join all threads to make sure they are synchronized
24297 dprintf(3, ("Restarting for relocation"));
24298 gc_t_join.restart();
24299 #endif //MULTIPLE_HEAPS
24302 dprintf(3,("Relocating roots"));
24303 GCScan::GcScanRoots(GCHeap::Relocate,
24304 condemned_gen_number, max_generation, &sc);
24306 verify_pins_with_post_plug_info("after reloc stack");
24308 #ifdef BACKGROUND_GC
24309 if (recursive_gc_sync::background_running_p())
24311 scan_background_roots (GCHeap::Relocate, heap_number, &sc);
24313 #endif //BACKGROUND_GC
24315 if (condemned_gen_number != max_generation)
24317 dprintf(3,("Relocating cross generation pointers"));
24318 mark_through_cards_for_segments (&gc_heap::relocate_address, TRUE);
24319 verify_pins_with_post_plug_info("after reloc cards");
24321 if (condemned_gen_number != max_generation)
24323 dprintf(3,("Relocating cross generation pointers for large objects"));
24324 mark_through_cards_for_large_objects (&gc_heap::relocate_address, TRUE);
24328 #ifdef FEATURE_LOH_COMPACTION
24329 if (loh_compacted_p)
24331 assert (settings.condemned_generation == max_generation);
24332 relocate_in_loh_compact();
24335 #endif //FEATURE_LOH_COMPACTION
24337 relocate_in_large_objects ();
24341 dprintf(3,("Relocating survivors"));
24342 relocate_survivors (condemned_gen_number,
24343 first_condemned_address);
24346 #ifdef FEATURE_PREMORTEM_FINALIZATION
24347 dprintf(3,("Relocating finalization data"));
24348 finalize_queue->RelocateFinalizationData (condemned_gen_number,
24350 #endif // FEATURE_PREMORTEM_FINALIZATION
24355 dprintf(3,("Relocating handle table"));
24356 GCScan::GcScanHandles(GCHeap::Relocate,
24357 condemned_gen_number, max_generation, &sc);
24360 #ifdef MULTIPLE_HEAPS
24361 //join all threads to make sure they are synchronized
24362 dprintf(3, ("Joining after end of relocation"));
24363 gc_t_join.join(this, gc_join_relocate_phase_done);
24365 #endif //MULTIPLE_HEAPS
24368 finish = GetCycleCount32();
24369 reloc_time = finish - start;
24372 dprintf(2,( "---- End of Relocate phase ----"));
24375 // This compares to see if tree is the current pinned plug and returns info
24376 // for this pinned plug. Also advances the pinned queue if that's the case.
24378 // We don't change the values of the plug info if tree is not the same as
24379 // the current pinned plug - the caller is responsible for setting the right
24380 // values to begin with.
24382 // POPO TODO: We are keeping this temporarily as this is also used by realloc
24383 // where it passes FALSE to deque_p, change it to use the same optimization
24384 // as relocate. Not as essential since realloc is already a slow path.
24385 mark* gc_heap::get_next_pinned_entry (uint8_t* tree,
24386 BOOL* has_pre_plug_info_p,
24387 BOOL* has_post_plug_info_p,
24390 if (!pinned_plug_que_empty_p())
24392 mark* oldest_entry = oldest_pin();
24393 uint8_t* oldest_plug = pinned_plug (oldest_entry);
24394 if (tree == oldest_plug)
24396 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24397 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24401 deque_pinned_plug();
24404 dprintf (3, ("found a pinned plug %Ix, pre: %d, post: %d",
24406 (*has_pre_plug_info_p ? 1 : 0),
24407 (*has_post_plug_info_p ? 1 : 0)));
24409 return oldest_entry;
24416 // This also deques the oldest entry and update the oldest plug
24417 mark* gc_heap::get_oldest_pinned_entry (BOOL* has_pre_plug_info_p,
24418 BOOL* has_post_plug_info_p)
24420 mark* oldest_entry = oldest_pin();
24421 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24422 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24424 deque_pinned_plug();
24425 update_oldest_pinned_plug();
24426 return oldest_entry;
24430 void gc_heap::copy_cards_range (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
24433 copy_cards_for_addresses (dest, src, len);
24435 clear_card_for_addresses (dest, dest + len);
24438 // POPO TODO: We should actually just recover the artifically made gaps here..because when we copy
24439 // we always copy the earlier plugs first which means we won't need the gap sizes anymore. This way
24440 // we won't need to individually recover each overwritten part of plugs.
24442 void gc_heap::gcmemcopy (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
24446 #ifdef BACKGROUND_GC
24447 if (current_c_gc_state == c_gc_state_marking)
24449 //TODO: should look to see whether we should consider changing this
24450 // to copy a consecutive region of the mark array instead.
24451 copy_mark_bits_for_addresses (dest, src, len);
24453 #endif //BACKGROUND_GC
24454 //dprintf(3,(" Memcopy [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
24455 dprintf(3,(" mc: [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
24456 memcopy (dest - plug_skew, src - plug_skew, len);
24457 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
24458 if (SoftwareWriteWatch::IsEnabledForGCHeap())
24460 // The ranges [src - plug_kew .. src[ and [src + len - plug_skew .. src + len[ are ObjHeaders, which don't have GC
24461 // references, and are not relevant for write watch. The latter range actually corresponds to the ObjHeader for the
24462 // object at (src + len), so it can be ignored anyway.
24463 SoftwareWriteWatch::SetDirtyRegion(dest, len - plug_skew);
24465 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
24466 copy_cards_range (dest, src, len, copy_cards_p);
24470 void gc_heap::compact_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, compact_args* args)
24473 uint8_t* reloc_plug = plug + args->last_plug_relocation;
24475 if (check_last_object_p)
24477 size += sizeof (gap_reloc_pair);
24478 mark* entry = args->pinned_plug_entry;
24480 if (args->is_shortened)
24482 assert (entry->has_post_plug_info());
24483 entry->swap_post_plug_and_saved();
24487 assert (entry->has_pre_plug_info());
24488 entry->swap_pre_plug_and_saved();
24492 int old_brick_entry = brick_table [brick_of (plug)];
24494 assert (node_relocation_distance (plug) == args->last_plug_relocation);
24496 #ifdef FEATURE_STRUCTALIGN
24497 ptrdiff_t alignpad = node_alignpad(plug);
24500 make_unused_array (reloc_plug - alignpad, alignpad);
24501 if (brick_of (reloc_plug - alignpad) != brick_of (reloc_plug))
24503 // The alignment padding is straddling one or more bricks;
24504 // it has to be the last "object" of its first brick.
24505 fix_brick_to_highest (reloc_plug - alignpad, reloc_plug);
24508 #else // FEATURE_STRUCTALIGN
24509 size_t unused_arr_size = 0;
24510 BOOL already_padded_p = FALSE;
24512 if (is_plug_padded (plug))
24514 already_padded_p = TRUE;
24515 clear_plug_padded (plug);
24516 unused_arr_size = Align (min_obj_size);
24518 #endif //SHORT_PLUGS
24519 if (node_realigned (plug))
24521 unused_arr_size += switch_alignment_size (already_padded_p);
24524 if (unused_arr_size != 0)
24526 make_unused_array (reloc_plug - unused_arr_size, unused_arr_size);
24528 if (brick_of (reloc_plug - unused_arr_size) != brick_of (reloc_plug))
24530 dprintf (3, ("fix B for padding: %Id: %Ix->%Ix",
24531 unused_arr_size, (reloc_plug - unused_arr_size), reloc_plug));
24532 // The alignment padding is straddling one or more bricks;
24533 // it has to be the last "object" of its first brick.
24534 fix_brick_to_highest (reloc_plug - unused_arr_size, reloc_plug);
24537 #endif // FEATURE_STRUCTALIGN
24540 if (is_plug_padded (plug))
24542 make_unused_array (reloc_plug - Align (min_obj_size), Align (min_obj_size));
24544 if (brick_of (reloc_plug - Align (min_obj_size)) != brick_of (reloc_plug))
24546 // The alignment padding is straddling one or more bricks;
24547 // it has to be the last "object" of its first brick.
24548 fix_brick_to_highest (reloc_plug - Align (min_obj_size), reloc_plug);
24551 #endif //SHORT_PLUGS
24553 gcmemcopy (reloc_plug, plug, size, args->copy_cards_p);
24555 if (args->check_gennum_p)
24557 int src_gennum = args->src_gennum;
24558 if (src_gennum == -1)
24560 src_gennum = object_gennum (plug);
24563 int dest_gennum = object_gennum_plan (reloc_plug);
24565 if (src_gennum < dest_gennum)
24567 generation_allocation_size (generation_of (dest_gennum)) += size;
24571 size_t current_reloc_brick = args->current_compacted_brick;
24573 if (brick_of (reloc_plug) != current_reloc_brick)
24575 dprintf (3, ("last reloc B: %Ix, current reloc B: %Ix",
24576 current_reloc_brick, brick_of (reloc_plug)));
24578 if (args->before_last_plug)
24580 dprintf (3,(" fixing last brick %Ix to point to last plug %Ix(%Ix)",
24581 current_reloc_brick,
24582 args->before_last_plug,
24583 (args->before_last_plug - brick_address (current_reloc_brick))));
24586 set_brick (current_reloc_brick,
24587 args->before_last_plug - brick_address (current_reloc_brick));
24590 current_reloc_brick = brick_of (reloc_plug);
24592 size_t end_brick = brick_of (reloc_plug + size-1);
24593 if (end_brick != current_reloc_brick)
24595 // The plug is straddling one or more bricks
24596 // It has to be the last plug of its first brick
24597 dprintf (3,("plug spanning multiple bricks, fixing first brick %Ix to %Ix(%Ix)",
24598 current_reloc_brick, (size_t)reloc_plug,
24599 (reloc_plug - brick_address (current_reloc_brick))));
24602 set_brick (current_reloc_brick,
24603 reloc_plug - brick_address (current_reloc_brick));
24605 // update all intervening brick
24606 size_t brick = current_reloc_brick + 1;
24607 dprintf (3,("setting intervening bricks %Ix->%Ix to -1",
24608 brick, (end_brick - 1)));
24609 while (brick < end_brick)
24611 set_brick (brick, -1);
24614 // code last brick offset as a plug address
24615 args->before_last_plug = brick_address (end_brick) -1;
24616 current_reloc_brick = end_brick;
24617 dprintf (3, ("setting before last to %Ix, last brick to %Ix",
24618 args->before_last_plug, current_reloc_brick));
24622 dprintf (3, ("still in the same brick: %Ix", end_brick));
24623 args->before_last_plug = reloc_plug;
24625 args->current_compacted_brick = current_reloc_brick;
24627 if (check_last_object_p)
24629 mark* entry = args->pinned_plug_entry;
24631 if (args->is_shortened)
24633 entry->swap_post_plug_and_saved();
24637 entry->swap_pre_plug_and_saved();
24642 void gc_heap::compact_in_brick (uint8_t* tree, compact_args* args)
24644 assert (tree != NULL);
24645 int left_node = node_left_child (tree);
24646 int right_node = node_right_child (tree);
24647 ptrdiff_t relocation = node_relocation_distance (tree);
24653 dprintf (3, ("B: L: %d->%Ix", left_node, (tree + left_node)));
24654 compact_in_brick ((tree + left_node), args);
24657 uint8_t* plug = tree;
24658 BOOL has_pre_plug_info_p = FALSE;
24659 BOOL has_post_plug_info_p = FALSE;
24661 if (tree == oldest_pinned_plug)
24663 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24664 &has_post_plug_info_p);
24665 assert (tree == pinned_plug (args->pinned_plug_entry));
24668 if (args->last_plug != 0)
24670 size_t gap_size = node_gap_size (tree);
24671 uint8_t* gap = (plug - gap_size);
24672 uint8_t* last_plug_end = gap;
24673 size_t last_plug_size = (last_plug_end - args->last_plug);
24674 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
24675 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
24677 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24678 if (!check_last_object_p)
24680 assert (last_plug_size >= Align (min_obj_size));
24683 compact_plug (args->last_plug, last_plug_size, check_last_object_p, args);
24687 assert (!has_pre_plug_info_p);
24690 dprintf (3, ("set args last plug to plug: %Ix, reloc: %Ix", plug, relocation));
24691 args->last_plug = plug;
24692 args->last_plug_relocation = relocation;
24693 args->is_shortened = has_post_plug_info_p;
24697 dprintf (3, ("B: R: %d->%Ix", right_node, (tree + right_node)));
24698 compact_in_brick ((tree + right_node), args);
24702 void gc_heap::recover_saved_pinned_info()
24704 reset_pinned_queue_bos();
24706 while (!(pinned_plug_que_empty_p()))
24708 mark* oldest_entry = oldest_pin();
24709 oldest_entry->recover_plug_info();
24710 #ifdef GC_CONFIG_DRIVEN
24711 if (oldest_entry->has_pre_plug_info() && oldest_entry->has_post_plug_info())
24712 record_interesting_data_point (idp_pre_and_post_pin);
24713 else if (oldest_entry->has_pre_plug_info())
24714 record_interesting_data_point (idp_pre_pin);
24715 else if (oldest_entry->has_post_plug_info())
24716 record_interesting_data_point (idp_post_pin);
24717 #endif //GC_CONFIG_DRIVEN
24719 deque_pinned_plug();
24723 void gc_heap::compact_phase (int condemned_gen_number,
24724 uint8_t* first_condemned_address,
24727 // %type% category = quote (compact);
24731 start = GetCycleCount32();
24733 generation* condemned_gen = generation_of (condemned_gen_number);
24734 uint8_t* start_address = first_condemned_address;
24735 size_t current_brick = brick_of (start_address);
24736 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24738 PREFIX_ASSUME(current_heap_segment != NULL);
24740 reset_pinned_queue_bos();
24741 update_oldest_pinned_plug();
24743 BOOL reused_seg = expand_reused_seg_p();
24746 for (int i = 1; i <= max_generation; i++)
24748 generation_allocation_size (generation_of (i)) = 0;
24752 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
24754 size_t end_brick = brick_of (end_address-1);
24756 args.last_plug = 0;
24757 args.before_last_plug = 0;
24758 args.current_compacted_brick = ~((size_t)1);
24759 args.is_shortened = FALSE;
24760 args.pinned_plug_entry = 0;
24761 args.copy_cards_p = (condemned_gen_number >= 1) || !clear_cards;
24762 args.check_gennum_p = reused_seg;
24763 if (args.check_gennum_p)
24765 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
24768 dprintf (2,("---- Compact Phase: %Ix(%Ix)----",
24769 first_condemned_address, brick_of (first_condemned_address)));
24771 #ifdef MULTIPLE_HEAPS
24773 if (gc_t_join.joined())
24775 #endif //MULTIPLE_HEAPS
24777 #ifdef MULTIPLE_HEAPS
24778 dprintf(3, ("Restarting for compaction"));
24779 gc_t_join.restart();
24781 #endif //MULTIPLE_HEAPS
24783 reset_pinned_queue_bos();
24785 #ifdef FEATURE_LOH_COMPACTION
24786 if (loh_compacted_p)
24790 #endif //FEATURE_LOH_COMPACTION
24792 if ((start_address < end_address) ||
24793 (condemned_gen_number == max_generation))
24797 if (current_brick > end_brick)
24799 if (args.last_plug != 0)
24801 dprintf (3, ("compacting last plug: %Ix", args.last_plug))
24802 compact_plug (args.last_plug,
24803 (heap_segment_allocated (current_heap_segment) - args.last_plug),
24808 if (heap_segment_next_rw (current_heap_segment))
24810 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24811 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24812 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24813 args.last_plug = 0;
24814 if (args.check_gennum_p)
24816 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
24822 if (args.before_last_plug !=0)
24824 dprintf (3, ("Fixing last brick %Ix to point to plug %Ix",
24825 args.current_compacted_brick, (size_t)args.before_last_plug));
24826 assert (args.current_compacted_brick != ~1u);
24827 set_brick (args.current_compacted_brick,
24828 args.before_last_plug - brick_address (args.current_compacted_brick));
24834 int brick_entry = brick_table [ current_brick ];
24835 dprintf (3, ("B: %Ix(%Ix)->%Ix",
24836 current_brick, (size_t)brick_entry, (brick_address (current_brick) + brick_entry - 1)));
24838 if (brick_entry >= 0)
24840 compact_in_brick ((brick_address (current_brick) + brick_entry -1),
24849 recover_saved_pinned_info();
24852 finish = GetCycleCount32();
24853 compact_time = finish - start;
24856 concurrent_print_time_delta ("compact end");
24858 dprintf(2,("---- End of Compact phase ----"));
24861 #ifdef MULTIPLE_HEAPS
24864 #pragma warning(push)
24865 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
24867 void gc_heap::gc_thread_stub (void* arg)
24869 gc_heap* heap = (gc_heap*)arg;
24870 if (!gc_thread_no_affinitize_p)
24872 GCThreadAffinity affinity;
24873 affinity.Group = GCThreadAffinity::None;
24874 affinity.Processor = GCThreadAffinity::None;
24876 // We are about to set affinity for GC threads. It is a good place to set up NUMA and
24877 // CPU groups because the process mask, processor number, and group number are all
24878 // readily available.
24879 if (GCToOSInterface::CanEnableGCCPUGroups())
24880 set_thread_group_affinity_for_heap(heap->heap_number, &affinity);
24882 set_thread_affinity_mask_for_heap(heap->heap_number, &affinity);
24884 if (!GCToOSInterface::SetThreadAffinity(&affinity))
24886 dprintf(1, ("Failed to set thread affinity for server GC thread"));
24890 // server GC threads run at a higher priority than normal.
24891 GCToOSInterface::BoostThreadPriority();
24892 _alloca (256*heap->heap_number);
24893 heap->gc_thread_function();
24896 #pragma warning(pop)
24899 #endif //MULTIPLE_HEAPS
24901 #ifdef BACKGROUND_GC
24904 #pragma warning(push)
24905 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
24907 void gc_heap::bgc_thread_stub (void* arg)
24909 gc_heap* heap = (gc_heap*)arg;
24910 heap->bgc_thread = GCToEEInterface::GetThread();
24911 assert(heap->bgc_thread != nullptr);
24912 heap->bgc_thread_function();
24915 #pragma warning(pop)
24918 #endif //BACKGROUND_GC
24920 /*------------------ Background GC ----------------------------*/
24922 #ifdef BACKGROUND_GC
24924 void gc_heap::background_drain_mark_list (int thread)
24926 UNREFERENCED_PARAMETER(thread);
24928 size_t saved_c_mark_list_index = c_mark_list_index;
24930 if (saved_c_mark_list_index)
24932 concurrent_print_time_delta ("SML");
24934 while (c_mark_list_index != 0)
24936 size_t current_index = c_mark_list_index - 1;
24937 uint8_t* o = c_mark_list [current_index];
24938 background_mark_object (o THREAD_NUMBER_ARG);
24939 c_mark_list_index--;
24941 if (saved_c_mark_list_index)
24944 concurrent_print_time_delta ("EML");
24947 fire_drain_mark_list_event (saved_c_mark_list_index);
24951 // The background GC version of scan_dependent_handles (see that method for a more in-depth comment).
24952 #ifdef MULTIPLE_HEAPS
24953 // Since we only scan dependent handles while we are stopped we'll never interfere with FGCs scanning
24954 // them. So we can use the same static variables.
24955 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
24957 // Whenever we call this method there may have been preceding object promotions. So set
24958 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
24959 // based on the how the scanning proceeded).
24960 s_fUnscannedPromotions = TRUE;
24962 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
24963 // the state of this thread's portion of the dependent handle table. That's because promotions on other
24964 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
24965 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
24966 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
24967 // as all the others or they'll get out of step).
24970 // The various worker threads are all currently racing in this code. We need to work out if at least
24971 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
24972 // dependent handle table when both of the following conditions apply:
24973 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
24974 // object happens to correspond to a primary in one of our handles we might potentially have to
24975 // promote the associated secondary).
24976 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
24978 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
24979 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
24980 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
24981 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
24982 // follows below. Note that we can't read this outside of the join since on any iteration apart from
24983 // the first threads will be racing between reading this value and completing their previous
24984 // iteration's table scan.
24986 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
24987 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
24988 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
24989 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
24990 // we're safely joined.
24991 if (GCScan::GcDhUnpromotedHandlesExist(sc))
24992 s_fUnpromotedHandles = TRUE;
24994 // Synchronize all the threads so we can read our state variables safely. The following shared
24995 // variable (indicating whether we should scan the tables or terminate the loop) will be set by a
24996 // single thread inside the join.
24997 bgc_t_join.join(this, gc_join_scan_dependent_handles);
24998 if (bgc_t_join.joined())
25000 // We're synchronized so it's safe to read our shared state variables. We update another shared
25001 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
25002 // the loop. We scan if there has been at least one object promotion since last time and at least
25003 // one thread has a dependent handle table with a potential handle promotion possible.
25004 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
25006 // Reset our shared state variables (ready to be set again on this scan or with a good initial
25007 // value for the next call if we're terminating the loop).
25008 s_fUnscannedPromotions = FALSE;
25009 s_fUnpromotedHandles = FALSE;
25011 if (!s_fScanRequired)
25013 uint8_t* all_heaps_max = 0;
25014 uint8_t* all_heaps_min = MAX_PTR;
25016 for (i = 0; i < n_heaps; i++)
25018 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
25019 all_heaps_max = g_heaps[i]->background_max_overflow_address;
25020 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
25021 all_heaps_min = g_heaps[i]->background_min_overflow_address;
25023 for (i = 0; i < n_heaps; i++)
25025 g_heaps[i]->background_max_overflow_address = all_heaps_max;
25026 g_heaps[i]->background_min_overflow_address = all_heaps_min;
25030 // Restart all the workers.
25031 dprintf(2, ("Starting all gc thread mark stack overflow processing"));
25032 bgc_t_join.restart();
25035 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25036 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
25037 // global flag indicating that at least one object promotion may have occurred (the usual comment
25038 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
25039 // exit the method since we unconditionally set this variable on method entry anyway).
25040 if (background_process_mark_overflow (sc->concurrent))
25041 s_fUnscannedPromotions = TRUE;
25043 // If we decided that no scan was required we can terminate the loop now.
25044 if (!s_fScanRequired)
25047 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
25048 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
25049 // could miss noting the promotion of some primary objects).
25050 bgc_t_join.join(this, gc_join_rescan_dependent_handles);
25051 if (bgc_t_join.joined())
25053 // Restart all the workers.
25054 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
25055 bgc_t_join.restart();
25058 // If the portion of the dependent handle table managed by this worker has handles that could still be
25059 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
25060 // could require a rescan of handles on this or other workers.
25061 if (GCScan::GcDhUnpromotedHandlesExist(sc))
25062 if (GCScan::GcDhReScan(sc))
25063 s_fUnscannedPromotions = TRUE;
25067 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
25069 // Whenever we call this method there may have been preceding object promotions. So set
25070 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
25071 // based on the how the scanning proceeded).
25072 bool fUnscannedPromotions = true;
25074 // Scan dependent handles repeatedly until there are no further promotions that can be made or we made a
25075 // scan without performing any new promotions.
25076 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
25078 // On each iteration of the loop start with the assumption that no further objects have been promoted.
25079 fUnscannedPromotions = false;
25081 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25082 // being visible. If there was an overflow (background_process_mark_overflow returned true) then
25083 // additional objects now appear to be promoted and we should set the flag.
25084 if (background_process_mark_overflow (sc->concurrent))
25085 fUnscannedPromotions = true;
25087 // Perform the scan and set the flag if any promotions resulted.
25088 if (GCScan::GcDhReScan (sc))
25089 fUnscannedPromotions = true;
25092 // Perform a last processing of any overflowed mark stack.
25093 background_process_mark_overflow (sc->concurrent);
25095 #endif //MULTIPLE_HEAPS
25097 void gc_heap::recover_bgc_settings()
25099 if ((settings.condemned_generation < max_generation) && recursive_gc_sync::background_running_p())
25101 dprintf (2, ("restoring bgc settings"));
25102 settings = saved_bgc_settings;
25103 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
25107 void gc_heap::allow_fgc()
25109 assert (bgc_thread == GCToEEInterface::GetThread());
25110 bool bToggleGC = false;
25112 if (g_fSuspensionPending > 0)
25114 bToggleGC = GCToEEInterface::EnablePreemptiveGC();
25117 GCToEEInterface::DisablePreemptiveGC();
25122 BOOL gc_heap::should_commit_mark_array()
25124 return (recursive_gc_sync::background_running_p() || (current_bgc_state == bgc_initialized));
25127 void gc_heap::clear_commit_flag()
25129 generation* gen = generation_of (max_generation);
25130 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25135 if (gen != large_object_generation)
25137 gen = large_object_generation;
25138 seg = heap_segment_in_range (generation_start_segment (gen));
25146 if (seg->flags & heap_segment_flags_ma_committed)
25148 seg->flags &= ~heap_segment_flags_ma_committed;
25151 if (seg->flags & heap_segment_flags_ma_pcommitted)
25153 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25156 seg = heap_segment_next (seg);
25160 void gc_heap::clear_commit_flag_global()
25162 #ifdef MULTIPLE_HEAPS
25163 for (int i = 0; i < n_heaps; i++)
25165 g_heaps[i]->clear_commit_flag();
25168 clear_commit_flag();
25169 #endif //MULTIPLE_HEAPS
25172 void gc_heap::verify_mark_array_cleared (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25175 size_t markw = mark_word_of (begin);
25176 size_t markw_end = mark_word_of (end);
25178 while (markw < markw_end)
25180 if (mark_array_addr[markw])
25182 dprintf (1, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
25183 markw, mark_array_addr[markw], mark_word_address (markw)));
25189 UNREFERENCED_PARAMETER(begin);
25190 UNREFERENCED_PARAMETER(end);
25191 UNREFERENCED_PARAMETER(mark_array_addr);
25195 void gc_heap::verify_mark_array_cleared (heap_segment* seg, uint32_t* mark_array_addr)
25197 verify_mark_array_cleared (heap_segment_mem (seg), heap_segment_reserved (seg), mark_array_addr);
25200 BOOL gc_heap::commit_mark_array_new_seg (gc_heap* hp,
25202 uint32_t* new_card_table,
25203 uint8_t* new_lowest_address)
25205 UNREFERENCED_PARAMETER(hp); // compiler bug? -- this *is*, indeed, referenced
25207 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25208 uint8_t* end = heap_segment_reserved (seg);
25210 uint8_t* lowest = hp->background_saved_lowest_address;
25211 uint8_t* highest = hp->background_saved_highest_address;
25213 uint8_t* commit_start = NULL;
25214 uint8_t* commit_end = NULL;
25215 size_t commit_flag = 0;
25217 if ((highest >= start) &&
25220 if ((start >= lowest) && (end <= highest))
25222 dprintf (GC_TABLE_LOG, ("completely in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25223 start, end, lowest, highest));
25224 commit_flag = heap_segment_flags_ma_committed;
25228 dprintf (GC_TABLE_LOG, ("partially in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25229 start, end, lowest, highest));
25230 commit_flag = heap_segment_flags_ma_pcommitted;
25233 commit_start = max (lowest, start);
25234 commit_end = min (highest, end);
25236 if (!commit_mark_array_by_range (commit_start, commit_end, hp->mark_array))
25241 if (new_card_table == 0)
25243 new_card_table = g_gc_card_table;
25246 if (hp->card_table != new_card_table)
25248 if (new_lowest_address == 0)
25250 new_lowest_address = g_gc_lowest_address;
25253 uint32_t* ct = &new_card_table[card_word (gcard_of (new_lowest_address))];
25254 uint32_t* ma = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, new_lowest_address));
25256 dprintf (GC_TABLE_LOG, ("table realloc-ed: %Ix->%Ix, MA: %Ix->%Ix",
25257 hp->card_table, new_card_table,
25258 hp->mark_array, ma));
25260 if (!commit_mark_array_by_range (commit_start, commit_end, ma))
25266 seg->flags |= commit_flag;
25272 BOOL gc_heap::commit_mark_array_by_range (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25274 size_t beg_word = mark_word_of (begin);
25275 size_t end_word = mark_word_of (align_on_mark_word (end));
25276 uint8_t* commit_start = align_lower_page ((uint8_t*)&mark_array_addr[beg_word]);
25277 uint8_t* commit_end = align_on_page ((uint8_t*)&mark_array_addr[end_word]);
25278 size_t size = (size_t)(commit_end - commit_start);
25280 #ifdef SIMPLE_DPRINTF
25281 dprintf (GC_TABLE_LOG, ("range: %Ix->%Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), commit %Ix->%Ix(%Id)",
25283 beg_word, end_word,
25284 (end_word - beg_word) * sizeof (uint32_t),
25285 &mark_array_addr[beg_word],
25286 &mark_array_addr[end_word],
25287 (size_t)(&mark_array_addr[end_word] - &mark_array_addr[beg_word]),
25288 commit_start, commit_end,
25290 #endif //SIMPLE_DPRINTF
25292 if (GCToOSInterface::VirtualCommit (commit_start, size))
25294 // We can only verify the mark array is cleared from begin to end, the first and the last
25295 // page aren't necessarily all cleared 'cause they could be used by other segments or
25297 verify_mark_array_cleared (begin, end, mark_array_addr);
25302 dprintf (GC_TABLE_LOG, ("failed to commit %Id bytes", (end_word - beg_word) * sizeof (uint32_t)));
25307 BOOL gc_heap::commit_mark_array_with_check (heap_segment* seg, uint32_t* new_mark_array_addr)
25309 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25310 uint8_t* end = heap_segment_reserved (seg);
25312 #ifdef MULTIPLE_HEAPS
25313 uint8_t* lowest = heap_segment_heap (seg)->background_saved_lowest_address;
25314 uint8_t* highest = heap_segment_heap (seg)->background_saved_highest_address;
25316 uint8_t* lowest = background_saved_lowest_address;
25317 uint8_t* highest = background_saved_highest_address;
25318 #endif //MULTIPLE_HEAPS
25320 if ((highest >= start) &&
25323 start = max (lowest, start);
25324 end = min (highest, end);
25325 if (!commit_mark_array_by_range (start, end, new_mark_array_addr))
25334 BOOL gc_heap::commit_mark_array_by_seg (heap_segment* seg, uint32_t* mark_array_addr)
25336 dprintf (GC_TABLE_LOG, ("seg: %Ix->%Ix; MA: %Ix",
25338 heap_segment_reserved (seg),
25340 uint8_t* start = (heap_segment_read_only_p (seg) ? heap_segment_mem (seg) : (uint8_t*)seg);
25342 return commit_mark_array_by_range (start, heap_segment_reserved (seg), mark_array_addr);
25345 BOOL gc_heap::commit_mark_array_bgc_init (uint32_t* mark_array_addr)
25347 UNREFERENCED_PARAMETER(mark_array_addr);
25349 dprintf (GC_TABLE_LOG, ("BGC init commit: lowest: %Ix, highest: %Ix, mark_array: %Ix",
25350 lowest_address, highest_address, mark_array));
25352 generation* gen = generation_of (max_generation);
25353 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25358 if (gen != large_object_generation)
25360 gen = large_object_generation;
25361 seg = heap_segment_in_range (generation_start_segment (gen));
25369 dprintf (GC_TABLE_LOG, ("seg: %Ix, flags: %Id", seg, seg->flags));
25371 if (!(seg->flags & heap_segment_flags_ma_committed))
25373 // For ro segments they could always be only partially in range so we'd
25374 // be calling this at the beginning of every BGC. We are not making this
25375 // more efficient right now - ro segments are currently only used by redhawk.
25376 if (heap_segment_read_only_p (seg))
25378 if ((heap_segment_mem (seg) >= lowest_address) &&
25379 (heap_segment_reserved (seg) <= highest_address))
25381 if (commit_mark_array_by_seg (seg, mark_array))
25383 seg->flags |= heap_segment_flags_ma_committed;
25392 uint8_t* start = max (lowest_address, heap_segment_mem (seg));
25393 uint8_t* end = min (highest_address, heap_segment_reserved (seg));
25394 if (commit_mark_array_by_range (start, end, mark_array))
25396 seg->flags |= heap_segment_flags_ma_pcommitted;
25406 // For normal segments they are by design completely in range so just
25407 // commit the whole mark array for each seg.
25408 if (commit_mark_array_by_seg (seg, mark_array))
25410 if (seg->flags & heap_segment_flags_ma_pcommitted)
25412 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25414 seg->flags |= heap_segment_flags_ma_committed;
25423 seg = heap_segment_next (seg);
25429 // This function doesn't check the commit flag since it's for a new array -
25430 // the mark_array flag for these segments will remain the same.
25431 BOOL gc_heap::commit_new_mark_array (uint32_t* new_mark_array_addr)
25433 dprintf (GC_TABLE_LOG, ("commiting existing segs on MA %Ix", new_mark_array_addr));
25434 generation* gen = generation_of (max_generation);
25435 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25440 if (gen != large_object_generation)
25442 gen = large_object_generation;
25443 seg = heap_segment_in_range (generation_start_segment (gen));
25451 if (!commit_mark_array_with_check (seg, new_mark_array_addr))
25456 seg = heap_segment_next (seg);
25459 #ifdef MULTIPLE_HEAPS
25460 if (new_heap_segment)
25462 if (!commit_mark_array_with_check (new_heap_segment, new_mark_array_addr))
25467 #endif //MULTIPLE_HEAPS
25472 BOOL gc_heap::commit_new_mark_array_global (uint32_t* new_mark_array)
25474 #ifdef MULTIPLE_HEAPS
25475 for (int i = 0; i < n_heaps; i++)
25477 if (!g_heaps[i]->commit_new_mark_array (new_mark_array))
25483 if (!commit_new_mark_array (new_mark_array))
25487 #endif //MULTIPLE_HEAPS
25492 void gc_heap::decommit_mark_array_by_seg (heap_segment* seg)
25494 // if BGC is disabled (the finalize watchdog does this at shutdown), the mark array could have
25495 // been set to NULL.
25496 if (mark_array == NULL)
25501 dprintf (GC_TABLE_LOG, ("decommitting seg %Ix(%Ix), MA: %Ix", seg, seg->flags, mark_array));
25503 size_t flags = seg->flags;
25505 if ((flags & heap_segment_flags_ma_committed) ||
25506 (flags & heap_segment_flags_ma_pcommitted))
25508 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25509 uint8_t* end = heap_segment_reserved (seg);
25511 if (flags & heap_segment_flags_ma_pcommitted)
25513 start = max (lowest_address, start);
25514 end = min (highest_address, end);
25517 size_t beg_word = mark_word_of (start);
25518 size_t end_word = mark_word_of (align_on_mark_word (end));
25519 uint8_t* decommit_start = align_on_page ((uint8_t*)&mark_array[beg_word]);
25520 uint8_t* decommit_end = align_lower_page ((uint8_t*)&mark_array[end_word]);
25521 size_t size = (size_t)(decommit_end - decommit_start);
25523 #ifdef SIMPLE_DPRINTF
25524 dprintf (GC_TABLE_LOG, ("seg: %Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), decommit %Ix->%Ix(%Id)",
25526 beg_word, end_word,
25527 (end_word - beg_word) * sizeof (uint32_t),
25528 &mark_array[beg_word],
25529 &mark_array[end_word],
25530 (size_t)(&mark_array[end_word] - &mark_array[beg_word]),
25531 decommit_start, decommit_end,
25533 #endif //SIMPLE_DPRINTF
25535 if (decommit_start < decommit_end)
25537 if (!GCToOSInterface::VirtualDecommit (decommit_start, size))
25539 dprintf (GC_TABLE_LOG, ("GCToOSInterface::VirtualDecommit on %Ix for %Id bytes failed",
25540 decommit_start, size));
25541 assert (!"decommit failed");
25545 dprintf (GC_TABLE_LOG, ("decommited [%Ix for address [%Ix", beg_word, seg));
25549 void gc_heap::background_mark_phase ()
25551 verify_mark_array_cleared();
25554 sc.thread_number = heap_number;
25555 sc.promotion = TRUE;
25556 sc.concurrent = FALSE;
25559 BOOL cooperative_mode = TRUE;
25560 #ifndef MULTIPLE_HEAPS
25561 const int thread = heap_number;
25562 #endif //!MULTIPLE_HEAPS
25564 dprintf(2,("-(GC%d)BMark-", VolatileLoad(&settings.gc_index)));
25566 assert (settings.concurrent);
25571 start = GetCycleCount32();
25574 #ifdef FFIND_OBJECT
25575 if (gen0_must_clear_bricks > 0)
25576 gen0_must_clear_bricks--;
25577 #endif //FFIND_OBJECT
25579 background_soh_alloc_count = 0;
25580 background_loh_alloc_count = 0;
25581 bgc_overflow_count = 0;
25583 bpromoted_bytes (heap_number) = 0;
25584 static uint32_t num_sizedrefs = 0;
25586 background_min_overflow_address = MAX_PTR;
25587 background_max_overflow_address = 0;
25588 background_min_soh_overflow_address = MAX_PTR;
25589 background_max_soh_overflow_address = 0;
25590 processed_soh_overflow_p = FALSE;
25593 //set up the mark lists from g_mark_list
25594 assert (g_mark_list);
25595 mark_list = g_mark_list;
25596 //dont use the mark list for full gc
25597 //because multiple segments are more complex to handle and the list
25598 //is likely to overflow
25599 mark_list_end = &mark_list [0];
25600 mark_list_index = &mark_list [0];
25602 c_mark_list_index = 0;
25604 shigh = (uint8_t*) 0;
25607 generation* gen = generation_of (max_generation);
25609 dprintf(3,("BGC: stack marking"));
25610 sc.concurrent = TRUE;
25612 GCScan::GcScanRoots(background_promote_callback,
25613 max_generation, max_generation,
25618 dprintf(3,("BGC: finalization marking"));
25619 finalize_queue->GcScanRoots(background_promote_callback, heap_number, 0);
25622 size_t total_loh_size = generation_size (max_generation + 1);
25623 bgc_begin_loh_size = total_loh_size;
25624 bgc_alloc_spin_loh = 0;
25625 bgc_loh_size_increased = 0;
25626 bgc_loh_allocated_in_free = 0;
25627 size_t total_soh_size = generation_sizes (generation_of (max_generation));
25629 dprintf (GTC_LOG, ("BM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
25632 //concurrent_print_time_delta ("copying stack roots");
25633 concurrent_print_time_delta ("CS");
25635 FIRE_EVENT(BGC1stNonConEnd);
25637 expanded_in_fgc = FALSE;
25638 saved_overflow_ephemeral_seg = 0;
25639 current_bgc_state = bgc_reset_ww;
25641 // we don't need a join here - just whichever thread that gets here
25642 // first can change the states and call restart_vm.
25643 // this is not true - we can't let the EE run when we are scanning stack.
25644 // since we now allow reset ww to run concurrently and have a join for it,
25645 // we can do restart ee on the 1st thread that got here. Make sure we handle the
25646 // sizedref handles correctly.
25647 #ifdef MULTIPLE_HEAPS
25648 bgc_t_join.join(this, gc_join_restart_ee);
25649 if (bgc_t_join.joined())
25650 #endif //MULTIPLE_HEAPS
25652 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25653 // Resetting write watch for software write watch is pretty fast, much faster than for hardware write watch. Reset
25654 // can be done while the runtime is suspended or after the runtime is restarted, the preference was to reset while
25655 // the runtime is suspended. The reset for hardware write watch is done after the runtime is restarted below.
25657 concurrent_print_time_delta ("CRWW begin");
25659 #ifdef MULTIPLE_HEAPS
25660 for (int i = 0; i < n_heaps; i++)
25662 g_heaps[i]->reset_write_watch (FALSE);
25665 reset_write_watch (FALSE);
25666 #endif //MULTIPLE_HEAPS
25668 concurrent_print_time_delta ("CRWW");
25669 #endif //WRITE_WATCH
25670 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25672 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
25674 // this c_write is not really necessary because restart_vm
25675 // has an instruction that will flush the cpu cache (interlocked
25676 // or whatever) but we don't want to rely on that.
25677 dprintf (BGC_LOG, ("setting cm_in_progress"));
25678 c_write (cm_in_progress, TRUE);
25680 //restart all thread, doing the marking from the array
25681 assert (dont_restart_ee_p);
25682 dont_restart_ee_p = FALSE;
25685 GCToOSInterface::YieldThread (0);
25686 #ifdef MULTIPLE_HEAPS
25687 dprintf(3, ("Starting all gc threads for gc"));
25688 bgc_t_join.restart();
25689 #endif //MULTIPLE_HEAPS
25692 #ifdef MULTIPLE_HEAPS
25693 bgc_t_join.join(this, gc_join_after_reset);
25694 if (bgc_t_join.joined())
25695 #endif //MULTIPLE_HEAPS
25697 disable_preemptive (true);
25699 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25700 // When software write watch is enabled, resetting write watch is done while the runtime is suspended above. The
25701 // post-reset call to revisit_written_pages is only necessary for concurrent reset_write_watch, to discard dirtied
25702 // pages during the concurrent reset.
25705 concurrent_print_time_delta ("CRWW begin");
25707 #ifdef MULTIPLE_HEAPS
25708 for (int i = 0; i < n_heaps; i++)
25710 g_heaps[i]->reset_write_watch (TRUE);
25713 reset_write_watch (TRUE);
25714 #endif //MULTIPLE_HEAPS
25716 concurrent_print_time_delta ("CRWW");
25717 #endif //WRITE_WATCH
25719 #ifdef MULTIPLE_HEAPS
25720 for (int i = 0; i < n_heaps; i++)
25722 g_heaps[i]->revisit_written_pages (TRUE, TRUE);
25725 revisit_written_pages (TRUE, TRUE);
25726 #endif //MULTIPLE_HEAPS
25728 concurrent_print_time_delta ("CRW");
25729 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25731 #ifdef MULTIPLE_HEAPS
25732 for (int i = 0; i < n_heaps; i++)
25734 g_heaps[i]->current_bgc_state = bgc_mark_handles;
25737 current_bgc_state = bgc_mark_handles;
25738 #endif //MULTIPLE_HEAPS
25740 current_c_gc_state = c_gc_state_marking;
25742 enable_preemptive ();
25744 #ifdef MULTIPLE_HEAPS
25745 dprintf(3, ("Joining BGC threads after resetting writewatch"));
25746 bgc_t_join.restart();
25747 #endif //MULTIPLE_HEAPS
25750 disable_preemptive (true);
25752 if (num_sizedrefs > 0)
25754 GCScan::GcScanSizedRefs(background_promote, max_generation, max_generation, &sc);
25756 enable_preemptive ();
25758 #ifdef MULTIPLE_HEAPS
25759 bgc_t_join.join(this, gc_join_scan_sizedref_done);
25760 if (bgc_t_join.joined())
25762 dprintf(3, ("Done with marking all sized refs. Starting all bgc thread for marking other strong roots"));
25763 bgc_t_join.restart();
25765 #endif //MULTIPLE_HEAPS
25767 disable_preemptive (true);
25770 dprintf (3,("BGC: handle table marking"));
25771 GCScan::GcScanHandles(background_promote,
25772 max_generation, max_generation,
25774 //concurrent_print_time_delta ("concurrent marking handle table");
25775 concurrent_print_time_delta ("CRH");
25777 current_bgc_state = bgc_mark_stack;
25778 dprintf (2,("concurrent draining mark list"));
25779 background_drain_mark_list (thread);
25780 //concurrent_print_time_delta ("concurrent marking stack roots");
25781 concurrent_print_time_delta ("CRS");
25783 dprintf (2,("concurrent revisiting dirtied pages"));
25784 revisit_written_pages (TRUE);
25785 revisit_written_pages (TRUE);
25786 //concurrent_print_time_delta ("concurrent marking dirtied pages on LOH");
25787 concurrent_print_time_delta ("CRre");
25789 enable_preemptive ();
25791 #ifdef MULTIPLE_HEAPS
25792 bgc_t_join.join(this, gc_join_concurrent_overflow);
25793 if (bgc_t_join.joined())
25795 uint8_t* all_heaps_max = 0;
25796 uint8_t* all_heaps_min = MAX_PTR;
25798 for (i = 0; i < n_heaps; i++)
25800 dprintf (3, ("heap %d overflow max is %Ix, min is %Ix",
25802 g_heaps[i]->background_max_overflow_address,
25803 g_heaps[i]->background_min_overflow_address));
25804 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
25805 all_heaps_max = g_heaps[i]->background_max_overflow_address;
25806 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
25807 all_heaps_min = g_heaps[i]->background_min_overflow_address;
25809 for (i = 0; i < n_heaps; i++)
25811 g_heaps[i]->background_max_overflow_address = all_heaps_max;
25812 g_heaps[i]->background_min_overflow_address = all_heaps_min;
25814 dprintf(3, ("Starting all bgc threads after updating the overflow info"));
25815 bgc_t_join.restart();
25817 #endif //MULTIPLE_HEAPS
25819 disable_preemptive (true);
25821 dprintf (2, ("before CRov count: %d", bgc_overflow_count));
25822 bgc_overflow_count = 0;
25823 background_process_mark_overflow (TRUE);
25824 dprintf (2, ("after CRov count: %d", bgc_overflow_count));
25825 bgc_overflow_count = 0;
25826 //concurrent_print_time_delta ("concurrent processing mark overflow");
25827 concurrent_print_time_delta ("CRov");
25829 // Stop all threads, crawl all stacks and revisit changed pages.
25830 FIRE_EVENT(BGC1stConEnd);
25832 dprintf (2, ("Stopping the EE"));
25834 enable_preemptive ();
25836 #ifdef MULTIPLE_HEAPS
25837 bgc_t_join.join(this, gc_join_suspend_ee);
25838 if (bgc_t_join.joined())
25840 bgc_threads_sync_event.Reset();
25842 dprintf(3, ("Joining BGC threads for non concurrent final marking"));
25843 bgc_t_join.restart();
25845 #endif //MULTIPLE_HEAPS
25847 if (heap_number == 0)
25849 enter_spin_lock (&gc_lock);
25853 bgc_threads_sync_event.Set();
25857 bgc_threads_sync_event.Wait(INFINITE, FALSE);
25858 dprintf (2, ("bgc_threads_sync_event is signalled"));
25861 assert (settings.concurrent);
25862 assert (settings.condemned_generation == max_generation);
25864 dprintf (2, ("clearing cm_in_progress"));
25865 c_write (cm_in_progress, FALSE);
25867 bgc_alloc_lock->check();
25869 current_bgc_state = bgc_final_marking;
25871 //concurrent_print_time_delta ("concurrent marking ended");
25872 concurrent_print_time_delta ("CR");
25874 FIRE_EVENT(BGC2ndNonConBegin);
25876 mark_absorb_new_alloc();
25878 // We need a join here 'cause find_object would complain if the gen0
25879 // bricks of another heap haven't been fixed up. So we need to make sure
25880 // that every heap's gen0 bricks are fixed up before we proceed.
25881 #ifdef MULTIPLE_HEAPS
25882 bgc_t_join.join(this, gc_join_after_absorb);
25883 if (bgc_t_join.joined())
25885 dprintf(3, ("Joining BGC threads after absorb"));
25886 bgc_t_join.restart();
25888 #endif //MULTIPLE_HEAPS
25890 // give VM a chance to do work
25891 GCToEEInterface::GcBeforeBGCSweepWork();
25893 //reset the flag, indicating that the EE no longer expect concurrent
25895 sc.concurrent = FALSE;
25897 total_loh_size = generation_size (max_generation + 1);
25898 total_soh_size = generation_sizes (generation_of (max_generation));
25900 dprintf (GTC_LOG, ("FM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
25902 dprintf (2, ("nonconcurrent marking stack roots"));
25903 GCScan::GcScanRoots(background_promote,
25904 max_generation, max_generation,
25906 //concurrent_print_time_delta ("nonconcurrent marking stack roots");
25907 concurrent_print_time_delta ("NRS");
25909 // finalize_queue->EnterFinalizeLock();
25910 finalize_queue->GcScanRoots(background_promote, heap_number, 0);
25911 // finalize_queue->LeaveFinalizeLock();
25913 dprintf (2, ("nonconcurrent marking handle table"));
25914 GCScan::GcScanHandles(background_promote,
25915 max_generation, max_generation,
25917 //concurrent_print_time_delta ("nonconcurrent marking handle table");
25918 concurrent_print_time_delta ("NRH");
25920 dprintf (2,("---- (GC%d)final going through written pages ----", VolatileLoad(&settings.gc_index)));
25921 revisit_written_pages (FALSE);
25922 //concurrent_print_time_delta ("nonconcurrent revisit dirtied pages on LOH");
25923 concurrent_print_time_delta ("NRre LOH");
25925 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25926 #ifdef MULTIPLE_HEAPS
25927 bgc_t_join.join(this, gc_join_disable_software_write_watch);
25928 if (bgc_t_join.joined())
25929 #endif // MULTIPLE_HEAPS
25931 // The runtime is suspended, and we will be doing a final query of dirty pages, so pause tracking written pages to
25932 // avoid further perf penalty after the runtime is restarted
25933 SoftwareWriteWatch::DisableForGCHeap();
25935 #ifdef MULTIPLE_HEAPS
25936 dprintf(3, ("Restarting BGC threads after disabling software write watch"));
25937 bgc_t_join.restart();
25938 #endif // MULTIPLE_HEAPS
25940 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25942 dprintf (2, ("before NR 1st Hov count: %d", bgc_overflow_count));
25943 bgc_overflow_count = 0;
25945 // Dependent handles need to be scanned with a special algorithm (see the header comment on
25946 // scan_dependent_handles for more detail). We perform an initial scan without processing any mark
25947 // stack overflow. This is not guaranteed to complete the operation but in a common case (where there
25948 // are no dependent handles that are due to be collected) it allows us to optimize away further scans.
25949 // The call to background_scan_dependent_handles is what will cycle through more iterations if
25950 // required and will also perform processing of any mark stack overflow once the dependent handle
25951 // table has been fully promoted.
25952 dprintf (2, ("1st dependent handle scan and process mark overflow"));
25953 GCScan::GcDhInitialScan(background_promote, max_generation, max_generation, &sc);
25954 background_scan_dependent_handles (&sc);
25955 //concurrent_print_time_delta ("1st nonconcurrent dependent handle scan and process mark overflow");
25956 concurrent_print_time_delta ("NR 1st Hov");
25958 dprintf (2, ("after NR 1st Hov count: %d", bgc_overflow_count));
25959 bgc_overflow_count = 0;
25961 #ifdef MULTIPLE_HEAPS
25962 bgc_t_join.join(this, gc_join_null_dead_short_weak);
25963 if (bgc_t_join.joined())
25964 #endif //MULTIPLE_HEAPS
25966 GCToEEInterface::AfterGcScanRoots (max_generation, max_generation, &sc);
25968 #ifdef MULTIPLE_HEAPS
25969 dprintf(3, ("Joining BGC threads for short weak handle scan"));
25970 bgc_t_join.restart();
25971 #endif //MULTIPLE_HEAPS
25974 // null out the target of short weakref that were not promoted.
25975 GCScan::GcShortWeakPtrScan(background_promote, max_generation, max_generation,&sc);
25977 //concurrent_print_time_delta ("bgc GcShortWeakPtrScan");
25978 concurrent_print_time_delta ("NR GcShortWeakPtrScan");
25982 #ifdef MULTIPLE_HEAPS
25983 bgc_t_join.join(this, gc_join_scan_finalization);
25984 if (bgc_t_join.joined())
25986 dprintf(3, ("Joining BGC threads for finalization"));
25987 bgc_t_join.restart();
25989 #endif //MULTIPLE_HEAPS
25991 //Handle finalization.
25992 dprintf(3,("Marking finalization data"));
25993 //concurrent_print_time_delta ("bgc joined to mark finalization");
25994 concurrent_print_time_delta ("NRj");
25996 // finalize_queue->EnterFinalizeLock();
25997 finalize_queue->ScanForFinalization (background_promote, max_generation, FALSE, __this);
25998 // finalize_queue->LeaveFinalizeLock();
26000 concurrent_print_time_delta ("NRF");
26003 dprintf (2, ("before NR 2nd Hov count: %d", bgc_overflow_count));
26004 bgc_overflow_count = 0;
26006 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
26007 // for finalization. As before background_scan_dependent_handles will also process any mark stack
26009 dprintf (2, ("2nd dependent handle scan and process mark overflow"));
26010 background_scan_dependent_handles (&sc);
26011 //concurrent_print_time_delta ("2nd nonconcurrent dependent handle scan and process mark overflow");
26012 concurrent_print_time_delta ("NR 2nd Hov");
26014 #ifdef MULTIPLE_HEAPS
26015 bgc_t_join.join(this, gc_join_null_dead_long_weak);
26016 if (bgc_t_join.joined())
26018 dprintf(2, ("Joining BGC threads for weak pointer deletion"));
26019 bgc_t_join.restart();
26021 #endif //MULTIPLE_HEAPS
26023 // null out the target of long weakref that were not promoted.
26024 GCScan::GcWeakPtrScan (background_promote, max_generation, max_generation, &sc);
26025 concurrent_print_time_delta ("NR GcWeakPtrScan");
26027 #ifdef MULTIPLE_HEAPS
26028 bgc_t_join.join(this, gc_join_null_dead_syncblk);
26029 if (bgc_t_join.joined())
26030 #endif //MULTIPLE_HEAPS
26032 dprintf (2, ("calling GcWeakPtrScanBySingleThread"));
26033 // scan for deleted entries in the syncblk cache
26034 GCScan::GcWeakPtrScanBySingleThread (max_generation, max_generation, &sc);
26035 concurrent_print_time_delta ("NR GcWeakPtrScanBySingleThread");
26036 #ifdef MULTIPLE_HEAPS
26037 dprintf(2, ("Starting BGC threads for end of background mark phase"));
26038 bgc_t_join.restart();
26039 #endif //MULTIPLE_HEAPS
26042 gen0_bricks_cleared = FALSE;
26044 dprintf (2, ("end of bgc mark: loh: %d, soh: %d",
26045 generation_size (max_generation + 1),
26046 generation_sizes (generation_of (max_generation))));
26048 for (int gen_idx = max_generation; gen_idx <= (max_generation + 1); gen_idx++)
26050 generation* gen = generation_of (gen_idx);
26051 dynamic_data* dd = dynamic_data_of (gen_idx);
26052 dd_begin_data_size (dd) = generation_size (gen_idx) -
26053 (generation_free_list_space (gen) + generation_free_obj_space (gen)) -
26054 Align (size (generation_allocation_start (gen)));
26055 dd_survived_size (dd) = 0;
26056 dd_pinned_survived_size (dd) = 0;
26057 dd_artificial_pinned_survived_size (dd) = 0;
26058 dd_added_pinned_size (dd) = 0;
26061 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26062 PREFIX_ASSUME(seg != NULL);
26066 seg->flags &= ~heap_segment_flags_swept;
26068 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
26070 // This can't happen...
26074 if (seg == ephemeral_heap_segment)
26076 heap_segment_background_allocated (seg) = generation_allocation_start (generation_of (max_generation - 1));
26080 heap_segment_background_allocated (seg) = heap_segment_allocated (seg);
26083 dprintf (2, ("seg %Ix background allocated is %Ix",
26084 heap_segment_mem (seg),
26085 heap_segment_background_allocated (seg)));
26086 seg = heap_segment_next_rw (seg);
26089 // We need to void alloc contexts here 'cause while background_ephemeral_sweep is running
26090 // we can't let the user code consume the left over parts in these alloc contexts.
26091 repair_allocation_contexts (FALSE);
26094 finish = GetCycleCount32();
26095 mark_time = finish - start;
26098 dprintf (2, ("end of bgc mark: gen2 free list space: %d, free obj space: %d",
26099 generation_free_list_space (generation_of (max_generation)),
26100 generation_free_obj_space (generation_of (max_generation))));
26102 dprintf(2,("---- (GC%d)End of background mark phase ----", VolatileLoad(&settings.gc_index)));
26106 gc_heap::suspend_EE ()
26108 dprintf (2, ("suspend_EE"));
26109 #ifdef MULTIPLE_HEAPS
26110 gc_heap* hp = gc_heap::g_heaps[0];
26111 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26113 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26114 #endif //MULTIPLE_HEAPS
26117 #ifdef MULTIPLE_HEAPS
26119 gc_heap::bgc_suspend_EE ()
26121 for (int i = 0; i < n_heaps; i++)
26123 gc_heap::g_heaps[i]->reset_gc_done();
26126 dprintf (2, ("bgc_suspend_EE"));
26127 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26129 gc_started = FALSE;
26130 for (int i = 0; i < n_heaps; i++)
26132 gc_heap::g_heaps[i]->set_gc_done();
26137 gc_heap::bgc_suspend_EE ()
26141 dprintf (2, ("bgc_suspend_EE"));
26142 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26143 gc_started = FALSE;
26146 #endif //MULTIPLE_HEAPS
26149 gc_heap::restart_EE ()
26151 dprintf (2, ("restart_EE"));
26152 #ifdef MULTIPLE_HEAPS
26153 GCToEEInterface::RestartEE(FALSE);
26155 GCToEEInterface::RestartEE(FALSE);
26156 #endif //MULTIPLE_HEAPS
26159 inline uint8_t* gc_heap::high_page ( heap_segment* seg, BOOL concurrent_p)
26163 uint8_t* end = ((seg == ephemeral_heap_segment) ?
26164 generation_allocation_start (generation_of (max_generation-1)) :
26165 heap_segment_allocated (seg));
26166 return align_lower_page (end);
26170 return heap_segment_allocated (seg);
26174 void gc_heap::revisit_written_page (uint8_t* page,
26178 uint8_t*& last_page,
26179 uint8_t*& last_object,
26180 BOOL large_objects_p,
26181 size_t& num_marked_objects)
26183 UNREFERENCED_PARAMETER(seg);
26185 uint8_t* start_address = page;
26187 int align_const = get_alignment_constant (!large_objects_p);
26188 uint8_t* high_address = end;
26189 uint8_t* current_lowest_address = background_saved_lowest_address;
26190 uint8_t* current_highest_address = background_saved_highest_address;
26191 BOOL no_more_loop_p = FALSE;
26194 #ifndef MULTIPLE_HEAPS
26195 const int thread = heap_number;
26196 #endif //!MULTIPLE_HEAPS
26198 if (large_objects_p)
26204 if (((last_page + WRITE_WATCH_UNIT_SIZE) == page)
26205 || (start_address <= last_object))
26211 o = find_first_object (start_address, last_object);
26212 // We can visit the same object again, but on a different page.
26213 assert (o >= last_object);
26217 dprintf (3,("page %Ix start: %Ix, %Ix[ ",
26218 (size_t)page, (size_t)o,
26219 (size_t)(min (high_address, page + WRITE_WATCH_UNIT_SIZE))));
26221 while (o < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26225 if (concurrent_p && large_objects_p)
26227 bgc_alloc_lock->bgc_mark_set (o);
26229 if (((CObjectHeader*)o)->IsFree())
26231 s = unused_array_size (o);
26243 dprintf (3,("Considering object %Ix(%s)", (size_t)o, (background_object_marked (o, FALSE) ? "bm" : "nbm")));
26245 assert (Align (s) >= Align (min_obj_size));
26247 uint8_t* next_o = o + Align (s, align_const);
26249 if (next_o >= start_address)
26251 #ifdef MULTIPLE_HEAPS
26254 // We set last_object here for SVR BGC here because SVR BGC has more than
26255 // one GC thread. When we have more than one GC thread we would run into this
26256 // situation if we skipped unmarked objects:
26257 // bgc thread 1 calls GWW, and detect object X not marked so it would skip it
26259 // bgc thread 2 marks X and all its current children.
26260 // user thread comes along and dirties more (and later) pages in X.
26261 // bgc thread 1 calls GWW again and gets those later pages but it will not mark anything
26262 // on them because it had already skipped X. We need to detect that this object is now
26263 // marked and mark the children on the dirtied pages.
26264 // In the future if we have less BGC threads than we have heaps we should add
26265 // the check to the number of BGC threads.
26268 #endif //MULTIPLE_HEAPS
26270 if (contain_pointers (o) &&
26271 (!((o >= current_lowest_address) && (o < current_highest_address)) ||
26272 background_marked (o)))
26274 dprintf (3, ("going through %Ix", (size_t)o));
26275 go_through_object (method_table(o), o, s, poo, start_address, use_start, (o + s),
26276 if ((uint8_t*)poo >= min (high_address, page + WRITE_WATCH_UNIT_SIZE))
26278 no_more_loop_p = TRUE;
26281 uint8_t* oo = *poo;
26283 num_marked_objects++;
26284 background_mark_object (oo THREAD_NUMBER_ARG);
26289 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // see comment below
26291 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26292 ((CObjectHeader*)o)->IsFree() &&
26293 (next_o > min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26295 // We need to not skip the object here because of this corner scenario:
26296 // A large object was being allocated during BGC mark so we first made it
26297 // into a free object, then cleared its memory. In this loop we would detect
26298 // that it's a free object which normally we would skip. But by the next time
26299 // we call GetWriteWatch we could still be on this object and the object had
26300 // been made into a valid object and some of its memory was changed. We need
26301 // to be sure to process those written pages so we can't skip the object just
26304 // Similarly, when using software write watch, don't advance last_object when
26305 // the current object is a free object that spans beyond the current page or
26306 // high_address. Software write watch acquires gc_lock before the concurrent
26307 // GetWriteWatch() call during revisit_written_pages(). A foreground GC may
26308 // happen at that point and allocate from this free region, so when
26309 // revisit_written_pages() continues, it cannot skip now-valid objects in this
26311 no_more_loop_p = TRUE;
26316 if (concurrent_p && large_objects_p)
26318 bgc_alloc_lock->bgc_mark_done ();
26320 if (no_more_loop_p)
26327 #ifdef MULTIPLE_HEAPS
26330 assert (last_object < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)));
26333 #endif //MULTIPLE_HEAPS
26338 dprintf (3,("Last object: %Ix", (size_t)last_object));
26339 last_page = align_write_watch_lower_page (o);
26342 // When reset_only_p is TRUE, we should only reset pages that are in range
26343 // because we need to consider the segments or part of segments that were
26344 // allocated out of range all live.
26345 void gc_heap::revisit_written_pages (BOOL concurrent_p, BOOL reset_only_p)
26348 if (concurrent_p && !reset_only_p)
26350 current_bgc_state = bgc_revisit_soh;
26353 size_t total_dirtied_pages = 0;
26354 size_t total_marked_objects = 0;
26356 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26358 PREFIX_ASSUME(seg != NULL);
26360 bool reset_watch_state = !!concurrent_p;
26361 bool is_runtime_suspended = !concurrent_p;
26362 BOOL small_object_segments = TRUE;
26363 int align_const = get_alignment_constant (small_object_segments);
26369 if (small_object_segments)
26371 //switch to large segment
26372 if (concurrent_p && !reset_only_p)
26374 current_bgc_state = bgc_revisit_loh;
26379 dprintf (GTC_LOG, ("h%d: SOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26380 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26381 concurrent_print_time_delta (concurrent_p ? "CR SOH" : "NR SOH");
26382 total_dirtied_pages = 0;
26383 total_marked_objects = 0;
26386 small_object_segments = FALSE;
26387 //concurrent_print_time_delta (concurrent_p ? "concurrent marking dirtied pages on SOH" : "nonconcurrent marking dirtied pages on SOH");
26389 dprintf (3, ("now revisiting large object segments"));
26390 align_const = get_alignment_constant (small_object_segments);
26391 seg = heap_segment_rw (generation_start_segment (large_object_generation));
26393 PREFIX_ASSUME(seg != NULL);
26401 dprintf (GTC_LOG, ("h%d: tdp: %Id", heap_number, total_dirtied_pages));
26405 dprintf (GTC_LOG, ("h%d: LOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26406 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26411 uint8_t* base_address = (uint8_t*)heap_segment_mem (seg);
26412 //we need to truncate to the base of the page because
26413 //some newly allocated could exist beyond heap_segment_allocated
26414 //and if we reset the last page write watch status,
26415 // they wouldn't be guaranteed to be visited -> gc hole.
26416 uintptr_t bcount = array_size;
26417 uint8_t* last_page = 0;
26418 uint8_t* last_object = heap_segment_mem (seg);
26419 uint8_t* high_address = 0;
26421 BOOL skip_seg_p = FALSE;
26425 if ((heap_segment_mem (seg) >= background_saved_lowest_address) ||
26426 (heap_segment_reserved (seg) <= background_saved_highest_address))
26428 dprintf (3, ("h%d: sseg: %Ix(-%Ix)", heap_number,
26429 heap_segment_mem (seg), heap_segment_reserved (seg)));
26436 dprintf (3, ("looking at seg %Ix", (size_t)last_object));
26440 base_address = max (base_address, background_saved_lowest_address);
26441 dprintf (3, ("h%d: reset only starting %Ix", heap_number, base_address));
26444 dprintf (3, ("h%d: starting: %Ix, seg %Ix-%Ix", heap_number, base_address,
26445 heap_segment_mem (seg), heap_segment_reserved (seg)));
26452 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
26453 high_address = min (high_address, background_saved_highest_address);
26457 high_address = high_page (seg, concurrent_p);
26460 if ((base_address < high_address) &&
26461 (bcount >= array_size))
26463 ptrdiff_t region_size = high_address - base_address;
26464 dprintf (3, ("h%d: gw: [%Ix(%Id)", heap_number, (size_t)base_address, (size_t)region_size));
26466 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26467 // When the runtime is not suspended, it's possible for the table to be resized concurrently with the scan
26468 // for dirty pages below. Prevent that by synchronizing with grow_brick_card_tables(). When the runtime is
26469 // suspended, it's ok to scan for dirty pages concurrently from multiple background GC threads for disjoint
26471 if (!is_runtime_suspended)
26473 enter_spin_lock(&gc_lock);
26475 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26477 get_write_watch_for_gc_heap (reset_watch_state, base_address, region_size,
26478 (void**)background_written_addresses,
26479 &bcount, is_runtime_suspended);
26481 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26482 if (!is_runtime_suspended)
26484 leave_spin_lock(&gc_lock);
26486 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26490 total_dirtied_pages += bcount;
26492 dprintf (3, ("Found %d pages [%Ix, %Ix[",
26493 bcount, (size_t)base_address, (size_t)high_address));
26498 for (unsigned i = 0; i < bcount; i++)
26500 uint8_t* page = (uint8_t*)background_written_addresses[i];
26501 dprintf (3, ("looking at page %d at %Ix(h: %Ix)", i,
26502 (size_t)page, (size_t)high_address));
26503 if (page < high_address)
26505 //search for marked objects in the page
26506 revisit_written_page (page, high_address, concurrent_p,
26507 seg, last_page, last_object,
26508 !small_object_segments,
26509 total_marked_objects);
26513 dprintf (3, ("page %d at %Ix is >= %Ix!", i, (size_t)page, (size_t)high_address));
26514 assert (!"page shouldn't have exceeded limit");
26519 if (bcount >= array_size){
26520 base_address = background_written_addresses [array_size-1] + WRITE_WATCH_UNIT_SIZE;
26521 bcount = array_size;
26531 seg = heap_segment_next_rw (seg);
26534 #endif //WRITE_WATCH
26537 void gc_heap::background_grow_c_mark_list()
26539 assert (c_mark_list_index >= c_mark_list_length);
26540 BOOL should_drain_p = FALSE;
26542 #ifndef MULTIPLE_HEAPS
26543 const int thread = heap_number;
26544 #endif //!MULTIPLE_HEAPS
26546 dprintf (2, ("stack copy buffer overflow"));
26547 uint8_t** new_c_mark_list = 0;
26550 if (c_mark_list_length >= (SIZE_T_MAX / (2 * sizeof (uint8_t*))))
26552 should_drain_p = TRUE;
26556 new_c_mark_list = new (nothrow) uint8_t*[c_mark_list_length*2];
26557 if (new_c_mark_list == 0)
26559 should_drain_p = TRUE;
26563 if (should_drain_p)
26566 dprintf (2, ("No more memory for the stacks copy, draining.."));
26567 //drain the list by marking its elements
26568 background_drain_mark_list (thread);
26572 assert (new_c_mark_list);
26573 memcpy (new_c_mark_list, c_mark_list, c_mark_list_length*sizeof(uint8_t*));
26574 c_mark_list_length = c_mark_list_length*2;
26575 delete c_mark_list;
26576 c_mark_list = new_c_mark_list;
26580 void gc_heap::background_promote_callback (Object** ppObject, ScanContext* sc,
26583 UNREFERENCED_PARAMETER(sc);
26584 //in order to save space on the array, mark the object,
26585 //knowing that it will be visited later
26586 assert (settings.concurrent);
26588 THREAD_NUMBER_FROM_CONTEXT;
26589 #ifndef MULTIPLE_HEAPS
26590 const int thread = 0;
26591 #endif //!MULTIPLE_HEAPS
26593 uint8_t* o = (uint8_t*)*ppObject;
26600 gc_heap* hp = gc_heap::heap_of (o);
26602 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
26607 #ifdef INTERIOR_POINTERS
26608 if (flags & GC_CALL_INTERIOR)
26610 o = hp->find_object (o, hp->background_saved_lowest_address);
26614 #endif //INTERIOR_POINTERS
26616 #ifdef FEATURE_CONSERVATIVE_GC
26617 // For conservative GC, a value on stack may point to middle of a free object.
26618 // In this case, we don't need to promote the pointer.
26619 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
26623 #endif //FEATURE_CONSERVATIVE_GC
26626 ((CObjectHeader*)o)->Validate();
26629 dprintf (3, ("Concurrent Background Promote %Ix", (size_t)o));
26630 if (o && (size (o) > LARGE_OBJECT_SIZE))
26632 dprintf (3, ("Brc %Ix", (size_t)o));
26635 if (hpt->c_mark_list_index >= hpt->c_mark_list_length)
26637 hpt->background_grow_c_mark_list();
26639 dprintf (3, ("pushing %08x into mark_list", (size_t)o));
26640 hpt->c_mark_list [hpt->c_mark_list_index++] = o;
26642 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Background Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
26645 void gc_heap::mark_absorb_new_alloc()
26647 fix_allocation_contexts (FALSE);
26649 gen0_bricks_cleared = FALSE;
26651 clear_gen0_bricks();
26654 BOOL gc_heap::prepare_bgc_thread(gc_heap* gh)
26656 BOOL success = FALSE;
26657 BOOL thread_created = FALSE;
26658 dprintf (2, ("Preparing gc thread"));
26659 gh->bgc_threads_timeout_cs.Enter();
26660 if (!(gh->bgc_thread_running))
26662 dprintf (2, ("GC thread not runnning"));
26663 if ((gh->bgc_thread == 0) && create_bgc_thread(gh))
26666 thread_created = TRUE;
26671 dprintf (3, ("GC thread already running"));
26674 gh->bgc_threads_timeout_cs.Leave();
26677 FIRE_EVENT(GCCreateConcurrentThread_V1);
26682 BOOL gc_heap::create_bgc_thread(gc_heap* gh)
26684 assert (background_gc_done_event.IsValid());
26686 //dprintf (2, ("Creating BGC thread"));
26688 gh->bgc_thread_running = GCToEEInterface::CreateThread(gh->bgc_thread_stub, gh, true, ".NET Background GC");
26689 return gh->bgc_thread_running;
26692 BOOL gc_heap::create_bgc_threads_support (int number_of_heaps)
26695 dprintf (3, ("Creating concurrent GC thread for the first time"));
26696 if (!background_gc_done_event.CreateManualEventNoThrow(TRUE))
26700 if (!bgc_threads_sync_event.CreateManualEventNoThrow(FALSE))
26704 if (!ee_proceed_event.CreateAutoEventNoThrow(FALSE))
26708 if (!bgc_start_event.CreateManualEventNoThrow(FALSE))
26713 #ifdef MULTIPLE_HEAPS
26714 bgc_t_join.init (number_of_heaps, join_flavor_bgc);
26716 UNREFERENCED_PARAMETER(number_of_heaps);
26717 #endif //MULTIPLE_HEAPS
26725 if (background_gc_done_event.IsValid())
26727 background_gc_done_event.CloseEvent();
26729 if (bgc_threads_sync_event.IsValid())
26731 bgc_threads_sync_event.CloseEvent();
26733 if (ee_proceed_event.IsValid())
26735 ee_proceed_event.CloseEvent();
26737 if (bgc_start_event.IsValid())
26739 bgc_start_event.CloseEvent();
26746 BOOL gc_heap::create_bgc_thread_support()
26751 if (!gc_lh_block_event.CreateManualEventNoThrow(FALSE))
26756 //needs to have room for enough smallest objects fitting on a page
26757 parr = new (nothrow) uint8_t*[1 + OS_PAGE_SIZE / MIN_OBJECT_SIZE];
26763 make_c_mark_list (parr);
26771 if (gc_lh_block_event.IsValid())
26773 gc_lh_block_event.CloseEvent();
26780 int gc_heap::check_for_ephemeral_alloc()
26782 int gen = ((settings.reason == reason_oos_soh) ? (max_generation - 1) : -1);
26786 #ifdef MULTIPLE_HEAPS
26787 for (int heap_index = 0; heap_index < n_heaps; heap_index++)
26788 #endif //MULTIPLE_HEAPS
26790 for (int i = 0; i <= (max_generation - 1); i++)
26792 #ifdef MULTIPLE_HEAPS
26793 if (g_heaps[heap_index]->get_new_allocation (i) <= 0)
26795 if (get_new_allocation (i) <= 0)
26796 #endif //MULTIPLE_HEAPS
26798 gen = max (gen, i);
26809 // Wait for gc to finish sequential part
26810 void gc_heap::wait_to_proceed()
26812 assert (background_gc_done_event.IsValid());
26813 assert (bgc_start_event.IsValid());
26815 user_thread_wait(&ee_proceed_event, FALSE);
26818 // Start a new concurrent gc
26819 void gc_heap::start_c_gc()
26821 assert (background_gc_done_event.IsValid());
26822 assert (bgc_start_event.IsValid());
26824 //Need to make sure that the gc thread is in the right place.
26825 background_gc_done_event.Wait(INFINITE, FALSE);
26826 background_gc_done_event.Reset();
26827 bgc_start_event.Set();
26830 void gc_heap::do_background_gc()
26832 dprintf (2, ("starting a BGC"));
26833 #ifdef MULTIPLE_HEAPS
26834 for (int i = 0; i < n_heaps; i++)
26836 g_heaps[i]->init_background_gc();
26839 init_background_gc();
26840 #endif //MULTIPLE_HEAPS
26841 //start the background gc
26844 //wait until we get restarted by the BGC.
26848 void gc_heap::kill_gc_thread()
26850 //assert (settings.concurrent == FALSE);
26852 // We are doing a two-stage shutdown now.
26853 // In the first stage, we do minimum work, and call ExitProcess at the end.
26854 // In the secodn stage, we have the Loader lock and only one thread is
26855 // alive. Hence we do not need to kill gc thread.
26856 background_gc_done_event.CloseEvent();
26857 gc_lh_block_event.CloseEvent();
26858 bgc_start_event.CloseEvent();
26859 bgc_threads_timeout_cs.Destroy();
26861 recursive_gc_sync::shutdown();
26864 void gc_heap::bgc_thread_function()
26866 assert (background_gc_done_event.IsValid());
26867 assert (bgc_start_event.IsValid());
26869 dprintf (3, ("gc_thread thread starting..."));
26871 BOOL do_exit = FALSE;
26873 bool cooperative_mode = true;
26874 bgc_thread_id.SetToCurrentThread();
26875 dprintf (1, ("bgc_thread_id is set to %x", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging()));
26878 // Wait for work to do...
26879 dprintf (3, ("bgc thread: waiting..."));
26881 cooperative_mode = enable_preemptive ();
26882 //current_thread->m_fPreemptiveGCDisabled = 0;
26884 uint32_t result = bgc_start_event.Wait(
26886 #ifdef MULTIPLE_HEAPS
26890 #endif //MULTIPLE_HEAPS
26892 #ifdef MULTIPLE_HEAPS
26896 #endif //MULTIPLE_HEAPS
26899 dprintf (2, ("gc thread: finished waiting"));
26901 // not calling disable_preemptive here 'cause we
26902 // can't wait for GC complete here - RestartEE will be called
26903 // when we've done the init work.
26905 if (result == WAIT_TIMEOUT)
26907 // Should join the bgc threads and terminate all of them
26909 dprintf (1, ("GC thread timeout"));
26910 bgc_threads_timeout_cs.Enter();
26911 if (!keep_bgc_threads_p)
26913 dprintf (2, ("GC thread exiting"));
26914 bgc_thread_running = FALSE;
26916 bgc_thread_id.Clear();
26919 bgc_threads_timeout_cs.Leave();
26924 dprintf (3, ("GC thread needed, not exiting"));
26928 // if we signal the thread with no concurrent work to do -> exit
26929 if (!settings.concurrent)
26931 dprintf (3, ("no concurrent GC needed, exiting"));
26937 recursive_gc_sync::begin_background();
26938 dprintf (2, ("beginning of bgc: gen2 FL: %d, FO: %d, frag: %d",
26939 generation_free_list_space (generation_of (max_generation)),
26940 generation_free_obj_space (generation_of (max_generation)),
26941 dd_fragmentation (dynamic_data_of (max_generation))));
26945 current_bgc_state = bgc_not_in_process;
26948 //trace_gc = FALSE;
26951 enable_preemptive ();
26952 #ifdef MULTIPLE_HEAPS
26953 bgc_t_join.join(this, gc_join_done);
26954 if (bgc_t_join.joined())
26955 #endif //MULTIPLE_HEAPS
26957 enter_spin_lock (&gc_lock);
26958 dprintf (SPINLOCK_LOG, ("bgc Egc"));
26960 bgc_start_event.Reset();
26962 #ifdef MULTIPLE_HEAPS
26963 for (int gen = max_generation; gen <= (max_generation + 1); gen++)
26965 size_t desired_per_heap = 0;
26966 size_t total_desired = 0;
26969 for (int i = 0; i < n_heaps; i++)
26972 dd = hp->dynamic_data_of (gen);
26973 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
26974 if (temp_total_desired < total_desired)
26977 total_desired = (size_t)MAX_PTR;
26980 total_desired = temp_total_desired;
26983 desired_per_heap = Align ((total_desired/n_heaps), get_alignment_constant (FALSE));
26985 for (int i = 0; i < n_heaps; i++)
26987 hp = gc_heap::g_heaps[i];
26988 dd = hp->dynamic_data_of (gen);
26989 dd_desired_allocation (dd) = desired_per_heap;
26990 dd_gc_new_allocation (dd) = desired_per_heap;
26991 dd_new_allocation (dd) = desired_per_heap;
26994 #endif //MULTIPLE_HEAPS
26995 #ifdef MULTIPLE_HEAPS
26997 #endif //MULTIPLE_HEAPS
26999 c_write (settings.concurrent, FALSE);
27000 recursive_gc_sync::end_background();
27001 keep_bgc_threads_p = FALSE;
27002 background_gc_done_event.Set();
27004 dprintf (SPINLOCK_LOG, ("bgc Lgc"));
27005 leave_spin_lock (&gc_lock);
27006 #ifdef MULTIPLE_HEAPS
27007 dprintf(1, ("End of BGC - starting all BGC threads"));
27008 bgc_t_join.restart();
27009 #endif //MULTIPLE_HEAPS
27011 // We can't disable preempt here because there might've been a GC already
27012 // started and decided to do a BGC and waiting for a BGC thread to restart
27013 // vm. That GC will be waiting in wait_to_proceed and we are waiting for it
27014 // to restart the VM so we deadlock.
27015 //gc_heap::disable_preemptive (current_thread, TRUE);
27018 FIRE_EVENT(GCTerminateConcurrentThread_V1);
27020 dprintf (3, ("bgc_thread thread exiting"));
27024 #endif //BACKGROUND_GC
27026 //Clear the cards [start_card, end_card[
27027 void gc_heap::clear_cards (size_t start_card, size_t end_card)
27029 if (start_card < end_card)
27031 size_t start_word = card_word (start_card);
27032 size_t end_word = card_word (end_card);
27033 if (start_word < end_word)
27035 // Figure out the bit positions of the cards within their words
27036 unsigned bits = card_bit (start_card);
27037 card_table [start_word] &= lowbits (~0, bits);
27038 for (size_t i = start_word+1; i < end_word; i++)
27039 card_table [i] = 0;
27040 bits = card_bit (end_card);
27041 // Don't write beyond end_card (and possibly uncommitted card table space).
27044 card_table [end_word] &= highbits (~0, bits);
27049 // If the start and end cards are in the same word, just clear the appropriate card
27050 // bits in that word.
27051 card_table [start_word] &= (lowbits (~0, card_bit (start_card)) |
27052 highbits (~0, card_bit (end_card)));
27054 #ifdef VERYSLOWDEBUG
27055 size_t card = start_card;
27056 while (card < end_card)
27058 assert (! (card_set_p (card)));
27061 #endif //VERYSLOWDEBUG
27062 dprintf (3,("Cleared cards [%Ix:%Ix, %Ix:%Ix[",
27063 start_card, (size_t)card_address (start_card),
27064 end_card, (size_t)card_address (end_card)));
27068 void gc_heap::clear_card_for_addresses (uint8_t* start_address, uint8_t* end_address)
27070 size_t start_card = card_of (align_on_card (start_address));
27071 size_t end_card = card_of (align_lower_card (end_address));
27072 clear_cards (start_card, end_card);
27075 // copy [srccard, ...[ to [dst_card, end_card[
27076 // This will set the same bit twice. Can be optimized.
27078 void gc_heap::copy_cards (size_t dst_card,
27083 // If the range is empty, this function is a no-op - with the subtlety that
27084 // either of the accesses card_table[srcwrd] or card_table[dstwrd] could be
27085 // outside the committed region. To avoid the access, leave early.
27086 if (!(dst_card < end_card))
27089 unsigned int srcbit = card_bit (src_card);
27090 unsigned int dstbit = card_bit (dst_card);
27091 size_t srcwrd = card_word (src_card);
27092 size_t dstwrd = card_word (dst_card);
27093 unsigned int srctmp = card_table[srcwrd];
27094 unsigned int dsttmp = card_table[dstwrd];
27096 for (size_t card = dst_card; card < end_card; card++)
27098 if (srctmp & (1 << srcbit))
27099 dsttmp |= 1 << dstbit;
27101 dsttmp &= ~(1 << dstbit);
27103 if (!(++srcbit % 32))
27105 srctmp = card_table[++srcwrd];
27111 if (srctmp & (1 << srcbit))
27112 dsttmp |= 1 << dstbit;
27115 if (!(++dstbit % 32))
27117 card_table[dstwrd] = dsttmp;
27119 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27122 card_bundle_set(cardw_card_bundle(dstwrd));
27127 dsttmp = card_table[dstwrd];
27132 card_table[dstwrd] = dsttmp;
27134 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27137 card_bundle_set(cardw_card_bundle(dstwrd));
27142 void gc_heap::copy_cards_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27144 ptrdiff_t relocation_distance = src - dest;
27145 size_t start_dest_card = card_of (align_on_card (dest));
27146 size_t end_dest_card = card_of (dest + len - 1);
27147 size_t dest_card = start_dest_card;
27148 size_t src_card = card_of (card_address (dest_card)+relocation_distance);
27149 dprintf (3,("Copying cards [%Ix:%Ix->%Ix:%Ix, ",
27150 src_card, (size_t)src, dest_card, (size_t)dest));
27151 dprintf (3,(" %Ix->%Ix:%Ix[",
27152 (size_t)src+len, end_dest_card, (size_t)dest+len));
27154 dprintf (3, ("dest: %Ix, src: %Ix, len: %Ix, reloc: %Ix, align_on_card(dest) is %Ix",
27155 dest, src, len, relocation_distance, (align_on_card (dest))));
27157 dprintf (3, ("start_dest_card: %Ix (address: %Ix), end_dest_card: %Ix(addr: %Ix), card_of (dest): %Ix",
27158 start_dest_card, card_address (start_dest_card), end_dest_card, card_address (end_dest_card), card_of (dest)));
27160 //First card has two boundaries
27161 if (start_dest_card != card_of (dest))
27163 if ((card_of (card_address (start_dest_card) + relocation_distance) <= card_of (src + len - 1))&&
27164 card_set_p (card_of (card_address (start_dest_card) + relocation_distance)))
27166 dprintf (3, ("card_address (start_dest_card) + reloc is %Ix, card: %Ix(set), src+len-1: %Ix, card: %Ix",
27167 (card_address (start_dest_card) + relocation_distance),
27168 card_of (card_address (start_dest_card) + relocation_distance),
27170 card_of (src + len - 1)));
27172 dprintf (3, ("setting card: %Ix", card_of (dest)));
27173 set_card (card_of (dest));
27177 if (card_set_p (card_of (src)))
27178 set_card (card_of (dest));
27181 copy_cards (dest_card, src_card, end_dest_card,
27182 ((dest - align_lower_card (dest)) != (src - align_lower_card (src))));
27184 //Last card has two boundaries.
27185 if ((card_of (card_address (end_dest_card) + relocation_distance) >= card_of (src)) &&
27186 card_set_p (card_of (card_address (end_dest_card) + relocation_distance)))
27188 dprintf (3, ("card_address (end_dest_card) + reloc is %Ix, card: %Ix(set), src: %Ix, card: %Ix",
27189 (card_address (end_dest_card) + relocation_distance),
27190 card_of (card_address (end_dest_card) + relocation_distance),
27194 dprintf (3, ("setting card: %Ix", end_dest_card));
27195 set_card (end_dest_card);
27198 if (card_set_p (card_of (src + len - 1)))
27199 set_card (end_dest_card);
27201 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27202 card_bundles_set(cardw_card_bundle(card_word(card_of(dest))), cardw_card_bundle(align_cardw_on_bundle(card_word(end_dest_card))));
27206 #ifdef BACKGROUND_GC
27207 // this does not need the Interlocked version of mark_array_set_marked.
27208 void gc_heap::copy_mark_bits_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27210 dprintf (3, ("Copying mark_bits for addresses [%Ix->%Ix, %Ix->%Ix[",
27211 (size_t)src, (size_t)dest,
27212 (size_t)src+len, (size_t)dest+len));
27214 uint8_t* src_o = src;
27216 uint8_t* src_end = src + len;
27217 int align_const = get_alignment_constant (TRUE);
27218 ptrdiff_t reloc = dest - src;
27220 while (src_o < src_end)
27222 uint8_t* next_o = src_o + Align (size (src_o), align_const);
27224 if (background_object_marked (src_o, TRUE))
27226 dest_o = src_o + reloc;
27228 //if (background_object_marked (dest_o, FALSE))
27230 // dprintf (3, ("*%Ix shouldn't have already been marked!", (size_t)(dest_o)));
27231 // FATAL_GC_ERROR();
27234 background_mark (dest_o,
27235 background_saved_lowest_address,
27236 background_saved_highest_address);
27237 dprintf (3, ("bc*%Ix*bc, b*%Ix*b", (size_t)src_o, (size_t)(dest_o)));
27243 #endif //BACKGROUND_GC
27245 void gc_heap::fix_brick_to_highest (uint8_t* o, uint8_t* next_o)
27247 size_t new_current_brick = brick_of (o);
27248 set_brick (new_current_brick,
27249 (o - brick_address (new_current_brick)));
27250 size_t b = 1 + new_current_brick;
27251 size_t limit = brick_of (next_o);
27252 //dprintf(3,(" fixing brick %Ix to point to object %Ix, till %Ix(%Ix)",
27253 dprintf(3,("b:%Ix->%Ix-%Ix",
27254 new_current_brick, (size_t)o, (size_t)next_o));
27257 set_brick (b,(new_current_brick - b));
27262 // start can not be >= heap_segment_allocated for the segment.
27263 uint8_t* gc_heap::find_first_object (uint8_t* start, uint8_t* first_object)
27265 size_t brick = brick_of (start);
27267 //last_object == null -> no search shortcut needed
27268 if ((brick == brick_of (first_object) || (start <= first_object)))
27274 ptrdiff_t min_brick = (ptrdiff_t)brick_of (first_object);
27275 ptrdiff_t prev_brick = (ptrdiff_t)brick - 1;
27276 int brick_entry = 0;
27279 if (prev_brick < min_brick)
27283 if ((brick_entry = get_brick_entry(prev_brick)) >= 0)
27287 assert (! ((brick_entry == 0)));
27288 prev_brick = (brick_entry + prev_brick);
27291 o = ((prev_brick < min_brick) ? first_object :
27292 brick_address (prev_brick) + brick_entry - 1);
27293 assert (o <= start);
27296 assert (Align (size (o)) >= Align (min_obj_size));
27297 uint8_t* next_o = o + Align (size (o));
27298 size_t curr_cl = (size_t)next_o / brick_size;
27299 size_t min_cl = (size_t)first_object / brick_size;
27301 //dprintf (3,( "Looking for intersection with %Ix from %Ix", (size_t)start, (size_t)o));
27303 unsigned int n_o = 1;
27306 uint8_t* next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27308 while (next_o <= start)
27316 assert (Align (size (o)) >= Align (min_obj_size));
27317 next_o = o + Align (size (o));
27319 }while (next_o < next_b);
27321 if (((size_t)next_o / brick_size) != curr_cl)
27323 if (curr_cl >= min_cl)
27325 fix_brick_to_highest (o, next_o);
27327 curr_cl = (size_t) next_o / brick_size;
27329 next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27332 size_t bo = brick_of (o);
27333 //dprintf (3, ("Looked at %Id objects, fixing brick [%Ix-[%Ix",
27334 dprintf (3, ("%Id o, [%Ix-[%Ix",
27338 set_brick (bo, (o - brick_address(bo)));
27353 // Find the first non-zero card word between cardw and cardw_end.
27354 // The index of the word we find is returned in cardw.
27355 BOOL gc_heap::find_card_dword (size_t& cardw, size_t cardw_end)
27357 dprintf (3, ("gc: %d, find_card_dword cardw: %Ix, cardw_end: %Ix",
27358 dd_collection_count (dynamic_data_of (0)), cardw, cardw_end));
27360 if (card_bundles_enabled())
27362 size_t cardb = cardw_card_bundle (cardw);
27363 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (cardw_end));
27366 // Find a non-zero bundle
27367 while ((cardb < end_cardb) && (card_bundle_set_p (cardb) == 0))
27372 if (cardb == end_cardb)
27375 // We found a bundle, so go through its words and find a non-zero card word
27376 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb),cardw)];
27377 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1),cardw_end)];
27378 while ((card_word < card_word_end) && !(*card_word))
27383 if (card_word != card_word_end)
27385 cardw = (card_word - &card_table[0]);
27388 else if ((cardw <= card_bundle_cardw (cardb)) &&
27389 (card_word == &card_table [card_bundle_cardw (cardb+1)]))
27391 // a whole bundle was explored and is empty
27392 dprintf (3, ("gc: %d, find_card_dword clear bundle: %Ix cardw:[%Ix,%Ix[",
27393 dd_collection_count (dynamic_data_of (0)),
27394 cardb, card_bundle_cardw (cardb),
27395 card_bundle_cardw (cardb+1)));
27396 card_bundle_clear (cardb);
27404 uint32_t* card_word = &card_table[cardw];
27405 uint32_t* card_word_end = &card_table [cardw_end];
27407 while (card_word < card_word_end)
27409 if (*card_word != 0)
27411 cardw = (card_word - &card_table [0]);
27422 #endif //CARD_BUNDLE
27424 // Find cards that are set between two points in a card table.
27426 // card_table : The card table.
27427 // card : [in/out] As input, the card to start searching from.
27428 // As output, the first card that's set.
27429 // card_word_end : The card word at which to stop looking.
27430 // end_card : [out] The last card which is set.
27431 BOOL gc_heap::find_card(uint32_t* card_table,
27433 size_t card_word_end,
27436 uint32_t* last_card_word;
27437 uint32_t card_word_value;
27438 uint32_t bit_position;
27440 // Find the first card which is set
27441 last_card_word = &card_table [card_word (card)];
27442 bit_position = card_bit (card);
27443 card_word_value = (*last_card_word) >> bit_position;
27444 if (!card_word_value)
27448 // Using the card bundle, go through the remaining card words between here and
27449 // card_word_end until we find one that is non-zero.
27450 size_t lcw = card_word(card) + 1;
27451 if (gc_heap::find_card_dword (lcw, card_word_end) == FALSE)
27457 last_card_word = &card_table [lcw];
27458 card_word_value = *last_card_word;
27461 #else //CARD_BUNDLE
27462 // Go through the remaining card words between here and card_word_end until we find
27463 // one that is non-zero.
27468 while ((last_card_word < &card_table [card_word_end]) && !(*last_card_word));
27470 if (last_card_word < &card_table [card_word_end])
27472 card_word_value = *last_card_word;
27476 // We failed to find any non-zero card words before we got to card_word_end
27479 #endif //CARD_BUNDLE
27482 // Look for the lowest bit set
27483 if (card_word_value)
27485 while (!(card_word_value & 1))
27488 card_word_value = card_word_value / 2;
27492 // card is the card word index * card size + the bit index within the card
27493 card = (last_card_word - &card_table[0]) * card_word_width + bit_position;
27497 // Keep going until we get to an un-set card.
27499 card_word_value = card_word_value / 2;
27501 // If we reach the end of the card word and haven't hit a 0 yet, start going
27502 // card word by card word until we get to one that's not fully set (0xFFFF...)
27503 // or we reach card_word_end.
27504 if ((bit_position == card_word_width) && (last_card_word < &card_table [card_word_end]))
27508 card_word_value = *(++last_card_word);
27509 } while ((last_card_word < &card_table [card_word_end]) &&
27512 (card_word_value == (1 << card_word_width)-1)
27514 // if left shift count >= width of type,
27515 // gcc reports error.
27516 (card_word_value == ~0u)
27521 } while (card_word_value & 1);
27523 end_card = (last_card_word - &card_table [0])* card_word_width + bit_position;
27525 //dprintf (3, ("find_card: [%Ix, %Ix[ set", card, end_card));
27526 dprintf (3, ("fc: [%Ix, %Ix[", card, end_card));
27531 //because of heap expansion, computing end is complicated.
27532 uint8_t* compute_next_end (heap_segment* seg, uint8_t* low)
27534 if ((low >= heap_segment_mem (seg)) &&
27535 (low < heap_segment_allocated (seg)))
27538 return heap_segment_allocated (seg);
27542 gc_heap::compute_next_boundary (uint8_t* low, int gen_number,
27545 UNREFERENCED_PARAMETER(low);
27547 //when relocating, the fault line is the plan start of the younger
27548 //generation because the generation is promoted.
27549 if (relocating && (gen_number == (settings.condemned_generation + 1)))
27551 generation* gen = generation_of (gen_number - 1);
27552 uint8_t* gen_alloc = generation_plan_allocation_start (gen);
27553 assert (gen_alloc);
27558 assert (gen_number > settings.condemned_generation);
27559 return generation_allocation_start (generation_of (gen_number - 1 ));
27565 gc_heap::keep_card_live (uint8_t* o, size_t& n_gen,
27566 size_t& cg_pointers_found)
27569 if ((gc_low <= o) && (gc_high > o))
27573 #ifdef MULTIPLE_HEAPS
27576 gc_heap* hp = heap_of (o);
27579 if ((hp->gc_low <= o) &&
27586 #endif //MULTIPLE_HEAPS
27587 cg_pointers_found ++;
27588 dprintf (4, ("keep card live for %Ix", o));
27592 gc_heap::mark_through_cards_helper (uint8_t** poo, size_t& n_gen,
27593 size_t& cg_pointers_found,
27594 card_fn fn, uint8_t* nhigh,
27595 uint8_t* next_boundary)
27598 if ((gc_low <= *poo) && (gc_high > *poo))
27601 call_fn(fn) (poo THREAD_NUMBER_ARG);
27603 #ifdef MULTIPLE_HEAPS
27606 gc_heap* hp = heap_of_gc (*poo);
27609 if ((hp->gc_low <= *poo) &&
27610 (hp->gc_high > *poo))
27613 call_fn(fn) (poo THREAD_NUMBER_ARG);
27615 if ((fn == &gc_heap::relocate_address) ||
27616 ((hp->ephemeral_low <= *poo) &&
27617 (hp->ephemeral_high > *poo)))
27619 cg_pointers_found++;
27623 #endif //MULTIPLE_HEAPS
27624 if ((next_boundary <= *poo) && (nhigh > *poo))
27626 cg_pointers_found ++;
27627 dprintf (4, ("cg pointer %Ix found, %Id so far",
27628 (size_t)*poo, cg_pointers_found ));
27633 BOOL gc_heap::card_transition (uint8_t* po, uint8_t* end, size_t card_word_end,
27634 size_t& cg_pointers_found,
27635 size_t& n_eph, size_t& n_card_set,
27636 size_t& card, size_t& end_card,
27637 BOOL& foundp, uint8_t*& start_address,
27638 uint8_t*& limit, size_t& n_cards_cleared)
27640 dprintf (3, ("pointer %Ix past card %Ix", (size_t)po, (size_t)card));
27641 dprintf (3, ("ct: %Id cg", cg_pointers_found));
27642 BOOL passed_end_card_p = FALSE;
27645 if (cg_pointers_found == 0)
27647 //dprintf(3,(" Clearing cards [%Ix, %Ix[ ",
27648 dprintf(3,(" CC [%Ix, %Ix[ ",
27649 (size_t)card_address(card), (size_t)po));
27650 clear_cards (card, card_of(po));
27651 n_card_set -= (card_of (po) - card);
27652 n_cards_cleared += (card_of (po) - card);
27655 n_eph +=cg_pointers_found;
27656 cg_pointers_found = 0;
27657 card = card_of (po);
27658 if (card >= end_card)
27660 passed_end_card_p = TRUE;
27661 dprintf (3, ("card %Ix exceeding end_card %Ix",
27662 (size_t)card, (size_t)end_card));
27663 foundp = find_card (card_table, card, card_word_end, end_card);
27666 n_card_set+= end_card - card;
27667 start_address = card_address (card);
27668 dprintf (3, ("NewC: %Ix, start: %Ix, end: %Ix",
27669 (size_t)card, (size_t)start_address,
27670 (size_t)card_address (end_card)));
27672 limit = min (end, card_address (end_card));
27674 assert (!((limit == card_address (end_card))&&
27675 card_set_p (end_card)));
27678 return passed_end_card_p;
27681 void gc_heap::mark_through_cards_for_segments (card_fn fn, BOOL relocating)
27683 #ifdef BACKGROUND_GC
27684 dprintf (3, ("current_sweep_pos is %Ix, saved_sweep_ephemeral_seg is %Ix(%Ix)",
27685 current_sweep_pos, saved_sweep_ephemeral_seg, saved_sweep_ephemeral_start));
27687 heap_segment* soh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
27688 PREFIX_ASSUME(soh_seg != NULL);
27692 dprintf (3, ("seg %Ix, bgc_alloc: %Ix, alloc: %Ix",
27694 heap_segment_background_allocated (soh_seg),
27695 heap_segment_allocated (soh_seg)));
27697 soh_seg = heap_segment_next_rw (soh_seg);
27699 #endif //BACKGROUND_GC
27701 uint8_t* low = gc_low;
27702 uint8_t* high = gc_high;
27703 size_t end_card = 0;
27705 generation* oldest_gen = generation_of (max_generation);
27706 int curr_gen_number = max_generation;
27707 uint8_t* gen_boundary = generation_allocation_start(generation_of(curr_gen_number - 1));
27708 uint8_t* next_boundary = compute_next_boundary(gc_low, curr_gen_number, relocating);
27710 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
27711 PREFIX_ASSUME(seg != NULL);
27713 uint8_t* beg = generation_allocation_start (oldest_gen);
27714 uint8_t* end = compute_next_end (seg, low);
27715 uint8_t* last_object = beg;
27717 size_t cg_pointers_found = 0;
27719 size_t card_word_end = (card_of (align_on_card_word (end)) / card_word_width);
27723 size_t n_card_set = 0;
27724 uint8_t* nhigh = (relocating ? heap_segment_plan_allocated (ephemeral_heap_segment) : high);
27726 BOOL foundp = FALSE;
27727 uint8_t* start_address = 0;
27728 uint8_t* limit = 0;
27729 size_t card = card_of (beg);
27730 #ifdef BACKGROUND_GC
27731 BOOL consider_bgc_mark_p = FALSE;
27732 BOOL check_current_sweep_p = FALSE;
27733 BOOL check_saved_sweep_p = FALSE;
27734 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
27735 #endif //BACKGROUND_GC
27737 dprintf(3, ("CMs: %Ix->%Ix", (size_t)beg, (size_t)end));
27738 size_t total_cards_cleared = 0;
27742 if (card_of(last_object) > card)
27744 // cg means cross-generational
27745 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
27746 if (cg_pointers_found == 0)
27748 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)last_object));
27749 clear_cards (card, card_of(last_object));
27750 n_card_set -= (card_of (last_object) - card);
27751 total_cards_cleared += (card_of (last_object) - card);
27754 n_eph += cg_pointers_found;
27755 cg_pointers_found = 0;
27756 card = card_of (last_object);
27759 if (card >= end_card)
27761 // Find the first card that's set (between card and card_word_end)
27762 foundp = find_card(card_table, card, card_word_end, end_card);
27765 // We found card(s) set.
27766 n_card_set += end_card - card;
27767 start_address = max (beg, card_address (card));
27770 limit = min (end, card_address (end_card));
27773 if (!foundp || (last_object >= end) || (card_address (card) >= end))
27775 if (foundp && (cg_pointers_found == 0))
27777 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
27779 clear_cards (card, card_of (end));
27780 n_card_set -= (card_of (end) - card);
27781 total_cards_cleared += (card_of (end) - card);
27784 n_eph += cg_pointers_found;
27785 cg_pointers_found = 0;
27787 if ((seg = heap_segment_next_in_range (seg)) != 0)
27789 #ifdef BACKGROUND_GC
27790 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
27791 #endif //BACKGROUND_GC
27792 beg = heap_segment_mem (seg);
27793 end = compute_next_end (seg, low);
27794 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
27795 card = card_of (beg);
27806 // We've found a card and will now go through the objects in it.
27807 assert (card_set_p (card));
27809 uint8_t* o = last_object;
27810 o = find_first_object (start_address, last_object);
27811 // Never visit an object twice.
27812 assert (o >= last_object);
27814 //dprintf(3,("Considering card %Ix start object: %Ix, %Ix[ boundary: %Ix",
27815 dprintf(3, ("c: %Ix, o: %Ix, l: %Ix[ boundary: %Ix",
27816 card, (size_t)o, (size_t)limit, (size_t)gen_boundary));
27820 assert (Align (size (o)) >= Align (min_obj_size));
27821 size_t s = size (o);
27823 uint8_t* next_o = o + Align (s);
27826 if ((o >= gen_boundary) &&
27827 (seg == ephemeral_heap_segment))
27829 dprintf (3, ("switching gen boundary %Ix", (size_t)gen_boundary));
27831 assert ((curr_gen_number > 0));
27832 gen_boundary = generation_allocation_start
27833 (generation_of (curr_gen_number - 1));
27834 next_boundary = (compute_next_boundary
27835 (low, curr_gen_number, relocating));
27838 dprintf (4, ("|%Ix|", (size_t)o));
27840 if (next_o < start_address)
27845 #ifdef BACKGROUND_GC
27846 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
27850 #endif //BACKGROUND_GC
27852 #ifdef COLLECTIBLE_CLASS
27853 if (is_collectible(o))
27855 BOOL passed_end_card_p = FALSE;
27857 if (card_of (o) > card)
27859 passed_end_card_p = card_transition (o, end, card_word_end,
27863 foundp, start_address,
27864 limit, total_cards_cleared);
27867 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
27869 // card is valid and it covers the head of the object
27870 if (fn == &gc_heap::relocate_address)
27872 keep_card_live (o, n_gen, cg_pointers_found);
27876 uint8_t* class_obj = get_class_object (o);
27877 mark_through_cards_helper (&class_obj, n_gen,
27878 cg_pointers_found, fn,
27879 nhigh, next_boundary);
27883 if (passed_end_card_p)
27885 if (foundp && (card_address (card) < next_o))
27887 goto go_through_refs;
27889 else if (foundp && (start_address < limit))
27891 next_o = find_first_object (start_address, o);
27900 #endif //COLLECTIBLE_CLASS
27902 if (contain_pointers (o))
27904 dprintf(3,("Going through %Ix start_address: %Ix", (size_t)o, (size_t)start_address));
27907 dprintf (4, ("normal object path"));
27909 (method_table(o), o, s, poo,
27910 start_address, use_start, (o + s),
27912 dprintf (4, ("<%Ix>:%Ix", (size_t)poo, (size_t)*poo));
27913 if (card_of ((uint8_t*)poo) > card)
27915 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
27920 foundp, start_address,
27921 limit, total_cards_cleared);
27923 if (passed_end_card_p)
27925 if (foundp && (card_address (card) < next_o))
27929 if (ppstop <= (uint8_t**)start_address)
27931 else if (poo < (uint8_t**)start_address)
27932 {poo = (uint8_t**)start_address;}
27935 else if (foundp && (start_address < limit))
27937 next_o = find_first_object (start_address, o);
27945 mark_through_cards_helper (poo, n_gen,
27946 cg_pointers_found, fn,
27947 nhigh, next_boundary);
27954 if (((size_t)next_o / brick_size) != ((size_t) o / brick_size))
27956 if (brick_table [brick_of (o)] <0)
27957 fix_brick_to_highest (o, next_o);
27965 // compute the efficiency ratio of the card table
27968 generation_skip_ratio = ((n_eph > 400)? (int)(((float)n_gen / (float)n_eph) * 100) : 100);
27969 dprintf (3, ("Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
27970 n_eph, n_gen , n_card_set, total_cards_cleared, generation_skip_ratio));
27974 dprintf (3, ("R: Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
27975 n_gen, n_eph, n_card_set, total_cards_cleared, generation_skip_ratio));
27979 #ifdef SEG_REUSE_STATS
27980 size_t gc_heap::dump_buckets (size_t* ordered_indices, int count, size_t* total_size)
27982 size_t total_items = 0;
27984 for (int i = 0; i < count; i++)
27986 total_items += ordered_indices[i];
27987 *total_size += ordered_indices[i] << (MIN_INDEX_POWER2 + i);
27988 dprintf (SEG_REUSE_LOG_0, ("[%d]%4d 2^%2d", heap_number, ordered_indices[i], (MIN_INDEX_POWER2 + i)));
27990 dprintf (SEG_REUSE_LOG_0, ("[%d]Total %d items, total size is 0x%Ix", heap_number, total_items, *total_size));
27991 return total_items;
27993 #endif // SEG_REUSE_STATS
27995 void gc_heap::count_plug (size_t last_plug_size, uint8_t*& last_plug)
27997 // detect pinned plugs
27998 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
28000 deque_pinned_plug();
28001 update_oldest_pinned_plug();
28002 dprintf (3, ("dequed pin,now oldest pin is %Ix", pinned_plug (oldest_pin())));
28006 size_t plug_size = last_plug_size + Align(min_obj_size);
28007 BOOL is_padded = FALSE;
28010 plug_size += Align (min_obj_size);
28012 #endif //SHORT_PLUGS
28014 #ifdef RESPECT_LARGE_ALIGNMENT
28015 plug_size += switch_alignment_size (is_padded);
28016 #endif //RESPECT_LARGE_ALIGNMENT
28018 total_ephemeral_plugs += plug_size;
28019 size_t plug_size_power2 = round_up_power2 (plug_size);
28020 ordered_plug_indices[relative_index_power2_plug (plug_size_power2)]++;
28021 dprintf (SEG_REUSE_LOG_1, ("[%d]count_plug: adding 0x%Ix - %Id (2^%d) to ordered plug array",
28025 (relative_index_power2_plug (plug_size_power2) + MIN_INDEX_POWER2)));
28029 void gc_heap::count_plugs_in_brick (uint8_t* tree, uint8_t*& last_plug)
28031 assert ((tree != NULL));
28032 if (node_left_child (tree))
28034 count_plugs_in_brick (tree + node_left_child (tree), last_plug);
28037 if (last_plug != 0)
28039 uint8_t* plug = tree;
28040 size_t gap_size = node_gap_size (plug);
28041 uint8_t* gap = (plug - gap_size);
28042 uint8_t* last_plug_end = gap;
28043 size_t last_plug_size = (last_plug_end - last_plug);
28044 dprintf (3, ("tree: %Ix, last plug: %Ix, gap size: %Ix, gap: %Ix, last plug size: %Ix",
28045 tree, last_plug, gap_size, gap, last_plug_size));
28047 if (tree == oldest_pinned_plug)
28049 dprintf (3, ("tree %Ix is pinned, last plug is %Ix, size is %Ix",
28050 tree, last_plug, last_plug_size));
28051 mark* m = oldest_pin();
28052 if (m->has_pre_plug_info())
28054 last_plug_size += sizeof (gap_reloc_pair);
28055 dprintf (3, ("pin %Ix has pre plug, adjusting plug size to %Ix", tree, last_plug_size));
28058 // Can't assert here - if it's a pinned plug it can be less.
28059 //assert (last_plug_size >= Align (min_obj_size));
28061 count_plug (last_plug_size, last_plug);
28066 if (node_right_child (tree))
28068 count_plugs_in_brick (tree + node_right_child (tree), last_plug);
28072 void gc_heap::build_ordered_plug_indices ()
28074 memset (ordered_plug_indices, 0, sizeof(ordered_plug_indices));
28075 memset (saved_ordered_plug_indices, 0, sizeof(saved_ordered_plug_indices));
28077 uint8_t* start_address = generation_limit (max_generation);
28078 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
28079 size_t current_brick = brick_of (start_address);
28080 size_t end_brick = brick_of (end_address - 1);
28081 uint8_t* last_plug = 0;
28083 //Look for the right pinned plug to start from.
28084 reset_pinned_queue_bos();
28085 while (!pinned_plug_que_empty_p())
28087 mark* m = oldest_pin();
28088 if ((m->first >= start_address) && (m->first < end_address))
28090 dprintf (3, ("found a pin %Ix between %Ix and %Ix", m->first, start_address, end_address));
28095 deque_pinned_plug();
28098 update_oldest_pinned_plug();
28100 while (current_brick <= end_brick)
28102 int brick_entry = brick_table [ current_brick ];
28103 if (brick_entry >= 0)
28105 count_plugs_in_brick (brick_address (current_brick) + brick_entry -1, last_plug);
28113 count_plug (end_address - last_plug, last_plug);
28116 // we need to make sure that after fitting all the existing plugs, we
28117 // have big enough free space left to guarantee that the next allocation
28119 size_t extra_size = END_SPACE_AFTER_GC + Align (min_obj_size);
28120 total_ephemeral_plugs += extra_size;
28121 dprintf (SEG_REUSE_LOG_0, ("Making sure we can fit a large object after fitting all plugs"));
28122 ordered_plug_indices[relative_index_power2_plug (round_up_power2 (extra_size))]++;
28124 memcpy (saved_ordered_plug_indices, ordered_plug_indices, sizeof(ordered_plug_indices));
28126 #ifdef SEG_REUSE_STATS
28127 dprintf (SEG_REUSE_LOG_0, ("Plugs:"));
28128 size_t total_plug_power2 = 0;
28129 dump_buckets (ordered_plug_indices, MAX_NUM_BUCKETS, &total_plug_power2);
28130 dprintf (SEG_REUSE_LOG_0, ("plugs: 0x%Ix (rounded up to 0x%Ix (%d%%))",
28131 total_ephemeral_plugs,
28133 (total_ephemeral_plugs ?
28134 (total_plug_power2 * 100 / total_ephemeral_plugs) :
28136 dprintf (SEG_REUSE_LOG_0, ("-------------------"));
28137 #endif // SEG_REUSE_STATS
28140 void gc_heap::init_ordered_free_space_indices ()
28142 memset (ordered_free_space_indices, 0, sizeof(ordered_free_space_indices));
28143 memset (saved_ordered_free_space_indices, 0, sizeof(saved_ordered_free_space_indices));
28146 void gc_heap::trim_free_spaces_indices ()
28148 trimmed_free_space_index = -1;
28149 size_t max_count = max_free_space_items - 1;
28152 for (i = (MAX_NUM_BUCKETS - 1); i >= 0; i--)
28154 count += ordered_free_space_indices[i];
28156 if (count >= max_count)
28162 ptrdiff_t extra_free_space_items = count - max_count;
28164 if (extra_free_space_items > 0)
28166 ordered_free_space_indices[i] -= extra_free_space_items;
28167 free_space_items = max_count;
28168 trimmed_free_space_index = i;
28172 free_space_items = count;
28180 free_space_buckets = MAX_NUM_BUCKETS - i;
28182 for (--i; i >= 0; i--)
28184 ordered_free_space_indices[i] = 0;
28187 memcpy (saved_ordered_free_space_indices,
28188 ordered_free_space_indices,
28189 sizeof(ordered_free_space_indices));
28192 // We fit as many plugs as we can and update the number of plugs left and the number
28193 // of free spaces left.
28194 BOOL gc_heap::can_fit_in_spaces_p (size_t* ordered_blocks, int small_index, size_t* ordered_spaces, int big_index)
28196 assert (small_index <= big_index);
28197 assert (big_index < MAX_NUM_BUCKETS);
28199 size_t small_blocks = ordered_blocks[small_index];
28201 if (small_blocks == 0)
28206 size_t big_spaces = ordered_spaces[big_index];
28208 if (big_spaces == 0)
28213 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting %Id 2^%d plugs into %Id 2^%d free spaces",
28215 small_blocks, (small_index + MIN_INDEX_POWER2),
28216 big_spaces, (big_index + MIN_INDEX_POWER2)));
28218 size_t big_to_small = big_spaces << (big_index - small_index);
28220 ptrdiff_t extra_small_spaces = big_to_small - small_blocks;
28221 dprintf (SEG_REUSE_LOG_1, ("[%d]%d 2^%d spaces can fit %d 2^%d blocks",
28223 big_spaces, (big_index + MIN_INDEX_POWER2), big_to_small, (small_index + MIN_INDEX_POWER2)));
28224 BOOL can_fit = (extra_small_spaces >= 0);
28228 dprintf (SEG_REUSE_LOG_1, ("[%d]Can fit with %d 2^%d extras blocks",
28230 extra_small_spaces, (small_index + MIN_INDEX_POWER2)));
28235 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d spaces to 0", heap_number, (big_index + MIN_INDEX_POWER2)));
28236 ordered_spaces[big_index] = 0;
28237 if (extra_small_spaces > 0)
28239 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d blocks to 0", heap_number, (small_index + MIN_INDEX_POWER2)));
28240 ordered_blocks[small_index] = 0;
28241 for (i = small_index; i < big_index; i++)
28243 if (extra_small_spaces & 1)
28245 dprintf (SEG_REUSE_LOG_1, ("[%d]Increasing # of 2^%d spaces from %d to %d",
28247 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + 1)));
28248 ordered_spaces[i] += 1;
28250 extra_small_spaces >>= 1;
28253 dprintf (SEG_REUSE_LOG_1, ("[%d]Finally increasing # of 2^%d spaces from %d to %d",
28255 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + extra_small_spaces)));
28256 ordered_spaces[i] += extra_small_spaces;
28260 dprintf (SEG_REUSE_LOG_1, ("[%d]Decreasing # of 2^%d blocks from %d to %d",
28262 (small_index + MIN_INDEX_POWER2),
28263 ordered_blocks[small_index],
28264 (ordered_blocks[small_index] - big_to_small)));
28265 ordered_blocks[small_index] -= big_to_small;
28268 #ifdef SEG_REUSE_STATS
28270 dprintf (SEG_REUSE_LOG_1, ("[%d]Plugs became:", heap_number));
28271 dump_buckets (ordered_blocks, MAX_NUM_BUCKETS, &temp);
28273 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces became:", heap_number));
28274 dump_buckets (ordered_spaces, MAX_NUM_BUCKETS, &temp);
28275 #endif //SEG_REUSE_STATS
28280 // space_index gets updated to the biggest available space index.
28281 BOOL gc_heap::can_fit_blocks_p (size_t* ordered_blocks, int block_index, size_t* ordered_spaces, int* space_index)
28283 assert (*space_index >= block_index);
28285 while (!can_fit_in_spaces_p (ordered_blocks, block_index, ordered_spaces, *space_index))
28288 if (*space_index < block_index)
28297 BOOL gc_heap::can_fit_all_blocks_p (size_t* ordered_blocks, size_t* ordered_spaces, int count)
28299 #ifdef FEATURE_STRUCTALIGN
28300 // BARTOKTODO (4841): reenable when can_fit_in_spaces_p takes alignment requirements into account
28302 #endif // FEATURE_STRUCTALIGN
28303 int space_index = count - 1;
28304 for (int block_index = (count - 1); block_index >= 0; block_index--)
28306 if (!can_fit_blocks_p (ordered_blocks, block_index, ordered_spaces, &space_index))
28315 void gc_heap::build_ordered_free_spaces (heap_segment* seg)
28317 assert (bestfit_seg);
28319 //bestfit_seg->add_buckets (MAX_NUM_BUCKETS - free_space_buckets + MIN_INDEX_POWER2,
28320 // ordered_free_space_indices + (MAX_NUM_BUCKETS - free_space_buckets),
28321 // free_space_buckets,
28322 // free_space_items);
28324 bestfit_seg->add_buckets (MIN_INDEX_POWER2,
28325 ordered_free_space_indices,
28329 assert (settings.condemned_generation == max_generation);
28331 uint8_t* first_address = heap_segment_mem (seg);
28332 uint8_t* end_address = heap_segment_reserved (seg);
28333 //look through the pinned plugs for relevant ones.
28334 //Look for the right pinned plug to start from.
28335 reset_pinned_queue_bos();
28337 // See comment in can_expand_into_p why we need (max_generation + 1).
28338 size_t eph_gen_starts = (Align (min_obj_size)) * (max_generation + 1);
28339 BOOL has_fit_gen_starts = FALSE;
28341 while (!pinned_plug_que_empty_p())
28344 if ((pinned_plug (m) >= first_address) &&
28345 (pinned_plug (m) < end_address) &&
28346 (pinned_len (m) >= eph_gen_starts))
28349 assert ((pinned_plug (m) - pinned_len (m)) == bestfit_first_pin);
28354 deque_pinned_plug();
28358 if (!pinned_plug_que_empty_p())
28360 bestfit_seg->add ((void*)m, TRUE, TRUE);
28361 deque_pinned_plug();
28363 has_fit_gen_starts = TRUE;
28366 while (!pinned_plug_que_empty_p() &&
28367 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
28369 bestfit_seg->add ((void*)m, TRUE, FALSE);
28370 deque_pinned_plug();
28374 if (commit_end_of_seg)
28376 if (!has_fit_gen_starts)
28378 assert (bestfit_first_pin == heap_segment_plan_allocated (seg));
28380 bestfit_seg->add ((void*)seg, FALSE, (!has_fit_gen_starts));
28384 bestfit_seg->check();
28388 BOOL gc_heap::try_best_fit (BOOL end_of_segment_p)
28390 if (!end_of_segment_p)
28392 trim_free_spaces_indices ();
28395 BOOL can_bestfit = can_fit_all_blocks_p (ordered_plug_indices,
28396 ordered_free_space_indices,
28399 return can_bestfit;
28402 BOOL gc_heap::best_fit (size_t free_space,
28403 size_t largest_free_space,
28404 size_t additional_space,
28405 BOOL* use_additional_space)
28407 dprintf (SEG_REUSE_LOG_0, ("gen%d: trying best fit mechanism", settings.condemned_generation));
28409 assert (!additional_space || (additional_space && use_additional_space));
28410 if (use_additional_space)
28412 *use_additional_space = FALSE;
28415 if (ordered_plug_indices_init == FALSE)
28417 total_ephemeral_plugs = 0;
28418 build_ordered_plug_indices();
28419 ordered_plug_indices_init = TRUE;
28423 memcpy (ordered_plug_indices, saved_ordered_plug_indices, sizeof(ordered_plug_indices));
28426 if (total_ephemeral_plugs == (END_SPACE_AFTER_GC + Align (min_obj_size)))
28428 dprintf (SEG_REUSE_LOG_0, ("No ephemeral plugs to realloc, done"));
28429 size_t empty_eph = (END_SPACE_AFTER_GC + Align (min_obj_size) + (Align (min_obj_size)) * (max_generation + 1));
28430 BOOL can_fit_empty_eph = (largest_free_space >= empty_eph);
28431 if (!can_fit_empty_eph)
28433 can_fit_empty_eph = (additional_space >= empty_eph);
28435 if (can_fit_empty_eph)
28437 *use_additional_space = TRUE;
28441 return can_fit_empty_eph;
28444 if ((total_ephemeral_plugs + approximate_new_allocation()) >= (free_space + additional_space))
28446 dprintf (SEG_REUSE_LOG_0, ("We won't have enough free space left in this segment after fitting, done"));
28450 if ((free_space + additional_space) == 0)
28452 dprintf (SEG_REUSE_LOG_0, ("No free space in this segment, done"));
28456 #ifdef SEG_REUSE_STATS
28457 dprintf (SEG_REUSE_LOG_0, ("Free spaces:"));
28458 size_t total_free_space_power2 = 0;
28459 size_t total_free_space_items =
28460 dump_buckets (ordered_free_space_indices,
28462 &total_free_space_power2);
28463 dprintf (SEG_REUSE_LOG_0, ("currently max free spaces is %Id", max_free_space_items));
28465 dprintf (SEG_REUSE_LOG_0, ("Ephemeral plugs: 0x%Ix, free space: 0x%Ix (rounded down to 0x%Ix (%Id%%)), additional free_space: 0x%Ix",
28466 total_ephemeral_plugs,
28468 total_free_space_power2,
28469 (free_space ? (total_free_space_power2 * 100 / free_space) : 0),
28470 additional_space));
28472 size_t saved_all_free_space_indices[MAX_NUM_BUCKETS];
28473 memcpy (saved_all_free_space_indices,
28474 ordered_free_space_indices,
28475 sizeof(saved_all_free_space_indices));
28477 #endif // SEG_REUSE_STATS
28479 if (total_ephemeral_plugs > (free_space + additional_space))
28484 use_bestfit = try_best_fit(FALSE);
28486 if (!use_bestfit && additional_space)
28488 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (additional_space));
28490 if (relative_free_space_index != -1)
28492 int relative_plug_index = 0;
28493 size_t plugs_to_fit = 0;
28495 for (relative_plug_index = (MAX_NUM_BUCKETS - 1); relative_plug_index >= 0; relative_plug_index--)
28497 plugs_to_fit = ordered_plug_indices[relative_plug_index];
28498 if (plugs_to_fit != 0)
28504 if ((relative_plug_index > relative_free_space_index) ||
28505 ((relative_plug_index == relative_free_space_index) &&
28506 (plugs_to_fit > 1)))
28508 #ifdef SEG_REUSE_STATS
28509 dprintf (SEG_REUSE_LOG_0, ("additional space is 2^%d but we stopped at %d 2^%d plug(s)",
28510 (relative_free_space_index + MIN_INDEX_POWER2),
28512 (relative_plug_index + MIN_INDEX_POWER2)));
28513 #endif // SEG_REUSE_STATS
28517 dprintf (SEG_REUSE_LOG_0, ("Adding end of segment (2^%d)", (relative_free_space_index + MIN_INDEX_POWER2)));
28518 ordered_free_space_indices[relative_free_space_index]++;
28519 use_bestfit = try_best_fit(TRUE);
28522 free_space_items++;
28523 // Since we might've trimmed away some of the free spaces we had, we should see
28524 // if we really need to use end of seg space - if it's the same or smaller than
28525 // the largest space we trimmed we can just add that one back instead of
28526 // using end of seg.
28527 if (relative_free_space_index > trimmed_free_space_index)
28529 *use_additional_space = TRUE;
28533 // If the addition space is <= than the last trimmed space, we
28534 // should just use that last trimmed space instead.
28535 saved_ordered_free_space_indices[trimmed_free_space_index]++;
28545 dprintf (SEG_REUSE_LOG_0, ("couldn't fit..."));
28547 #ifdef SEG_REUSE_STATS
28548 size_t saved_max = max_free_space_items;
28549 BOOL temp_bestfit = FALSE;
28551 dprintf (SEG_REUSE_LOG_0, ("----Starting experiment process----"));
28552 dprintf (SEG_REUSE_LOG_0, ("----Couldn't fit with max free items %Id", max_free_space_items));
28554 // TODO: need to take the end of segment into consideration.
28555 while (max_free_space_items <= total_free_space_items)
28557 max_free_space_items += max_free_space_items / 2;
28558 dprintf (SEG_REUSE_LOG_0, ("----Temporarily increasing max free spaces to %Id", max_free_space_items));
28559 memcpy (ordered_free_space_indices,
28560 saved_all_free_space_indices,
28561 sizeof(ordered_free_space_indices));
28562 if (try_best_fit(FALSE))
28564 temp_bestfit = TRUE;
28571 dprintf (SEG_REUSE_LOG_0, ("----With %Id max free spaces we could fit", max_free_space_items));
28575 dprintf (SEG_REUSE_LOG_0, ("----Tried all free spaces and still couldn't fit, lost too much space"));
28578 dprintf (SEG_REUSE_LOG_0, ("----Restoring max free spaces to %Id", saved_max));
28579 max_free_space_items = saved_max;
28580 #endif // SEG_REUSE_STATS
28581 if (free_space_items)
28583 max_free_space_items = min (MAX_NUM_FREE_SPACES, free_space_items * 2);
28584 max_free_space_items = max (max_free_space_items, MIN_NUM_FREE_SPACES);
28588 max_free_space_items = MAX_NUM_FREE_SPACES;
28592 dprintf (SEG_REUSE_LOG_0, ("Adjusted number of max free spaces to %Id", max_free_space_items));
28593 dprintf (SEG_REUSE_LOG_0, ("------End of best fitting process------\n"));
28595 return use_bestfit;
28598 BOOL gc_heap::process_free_space (heap_segment* seg,
28600 size_t min_free_size,
28601 size_t min_cont_size,
28602 size_t* total_free_space,
28603 size_t* largest_free_space)
28605 *total_free_space += free_space;
28606 *largest_free_space = max (*largest_free_space, free_space);
28608 #ifdef SIMPLE_DPRINTF
28609 dprintf (SEG_REUSE_LOG_1, ("free space len: %Ix, total free space: %Ix, largest free space: %Ix",
28610 free_space, *total_free_space, *largest_free_space));
28611 #endif //SIMPLE_DPRINTF
28613 if ((*total_free_space >= min_free_size) && (*largest_free_space >= min_cont_size))
28615 #ifdef SIMPLE_DPRINTF
28616 dprintf (SEG_REUSE_LOG_0, ("(gen%d)total free: %Ix(min: %Ix), largest free: %Ix(min: %Ix). Found segment %Ix to reuse without bestfit",
28617 settings.condemned_generation,
28618 *total_free_space, min_free_size, *largest_free_space, min_cont_size,
28621 UNREFERENCED_PARAMETER(seg);
28622 #endif //SIMPLE_DPRINTF
28626 int free_space_index = relative_index_power2_free_space (round_down_power2 (free_space));
28627 if (free_space_index != -1)
28629 ordered_free_space_indices[free_space_index]++;
28634 BOOL gc_heap::expand_reused_seg_p()
28636 BOOL reused_seg = FALSE;
28637 int heap_expand_mechanism = gc_data_per_heap.get_mechanism (gc_heap_expand);
28638 if ((heap_expand_mechanism == expand_reuse_bestfit) ||
28639 (heap_expand_mechanism == expand_reuse_normal))
28647 BOOL gc_heap::can_expand_into_p (heap_segment* seg, size_t min_free_size, size_t min_cont_size,
28648 allocator* gen_allocator)
28650 min_cont_size += END_SPACE_AFTER_GC;
28651 use_bestfit = FALSE;
28652 commit_end_of_seg = FALSE;
28653 bestfit_first_pin = 0;
28654 uint8_t* first_address = heap_segment_mem (seg);
28655 uint8_t* end_address = heap_segment_reserved (seg);
28656 size_t end_extra_space = end_space_after_gc();
28658 if ((heap_segment_reserved (seg) - end_extra_space) <= heap_segment_plan_allocated (seg))
28660 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: can't use segment [%Ix %Ix, has less than %d bytes at the end",
28661 first_address, end_address, end_extra_space));
28665 end_address -= end_extra_space;
28667 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p(gen%d): min free: %Ix, min continuous: %Ix",
28668 settings.condemned_generation, min_free_size, min_cont_size));
28669 size_t eph_gen_starts = eph_gen_starts_size;
28671 if (settings.condemned_generation == max_generation)
28673 size_t free_space = 0;
28674 size_t largest_free_space = free_space;
28675 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen2: testing segment [%Ix %Ix", first_address, end_address));
28676 //Look through the pinned plugs for relevant ones and Look for the right pinned plug to start from.
28677 //We are going to allocate the generation starts in the 1st free space,
28678 //so start from the first free space that's big enough for gen starts and a min object size.
28679 // If we see a free space that is >= gen starts but < gen starts + min obj size we just don't use it -
28680 // we could use it by allocating the last generation start a bit bigger but
28681 // the complexity isn't worth the effort (those plugs are from gen2
28682 // already anyway).
28683 reset_pinned_queue_bos();
28685 BOOL has_fit_gen_starts = FALSE;
28687 init_ordered_free_space_indices ();
28688 while (!pinned_plug_que_empty_p())
28691 if ((pinned_plug (m) >= first_address) &&
28692 (pinned_plug (m) < end_address) &&
28693 (pinned_len (m) >= (eph_gen_starts + Align (min_obj_size))))
28699 deque_pinned_plug();
28703 if (!pinned_plug_que_empty_p())
28705 bestfit_first_pin = pinned_plug (m) - pinned_len (m);
28707 if (process_free_space (seg,
28708 pinned_len (m) - eph_gen_starts,
28709 min_free_size, min_cont_size,
28710 &free_space, &largest_free_space))
28715 deque_pinned_plug();
28717 has_fit_gen_starts = TRUE;
28720 dprintf (3, ("first pin is %Ix", pinned_plug (m)));
28722 //tally up free space
28723 while (!pinned_plug_que_empty_p() &&
28724 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
28726 dprintf (3, ("looking at pin %Ix", pinned_plug (m)));
28727 if (process_free_space (seg,
28729 min_free_size, min_cont_size,
28730 &free_space, &largest_free_space))
28735 deque_pinned_plug();
28739 //try to find space at the end of the segment.
28740 size_t end_space = (end_address - heap_segment_plan_allocated (seg));
28741 size_t additional_space = ((min_free_size > free_space) ? (min_free_size - free_space) : 0);
28742 dprintf (SEG_REUSE_LOG_0, ("end space: %Ix; additional: %Ix", end_space, additional_space));
28743 if (end_space >= additional_space)
28745 BOOL can_fit = TRUE;
28746 commit_end_of_seg = TRUE;
28748 if (largest_free_space < min_cont_size)
28750 if (end_space >= min_cont_size)
28752 additional_space = max (min_cont_size, additional_space);
28753 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg for eph",
28758 if (settings.concurrent)
28761 commit_end_of_seg = FALSE;
28765 size_t additional_space_bestfit = additional_space;
28766 if (!has_fit_gen_starts)
28768 if (additional_space_bestfit < (eph_gen_starts + Align (min_obj_size)))
28770 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, gen starts not allocated yet and end space is too small: %Id",
28771 additional_space_bestfit));
28775 bestfit_first_pin = heap_segment_plan_allocated (seg);
28776 additional_space_bestfit -= eph_gen_starts;
28779 can_fit = best_fit (free_space,
28780 largest_free_space,
28781 additional_space_bestfit,
28782 &commit_end_of_seg);
28786 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse with bestfit, %s committing end of seg",
28787 seg, (commit_end_of_seg ? "with" : "without")));
28791 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
28798 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg", seg));
28801 assert (additional_space <= end_space);
28802 if (commit_end_of_seg)
28804 if (!grow_heap_segment (seg, heap_segment_plan_allocated (seg) + additional_space))
28806 dprintf (2, ("Couldn't commit end of segment?!"));
28807 use_bestfit = FALSE;
28814 // We increase the index here because growing heap segment could create a discrepency with
28815 // the additional space we used (could be bigger).
28816 size_t free_space_end_of_seg =
28817 heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
28818 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (free_space_end_of_seg));
28819 saved_ordered_free_space_indices[relative_free_space_index]++;
28825 memcpy (ordered_free_space_indices,
28826 saved_ordered_free_space_indices,
28827 sizeof(ordered_free_space_indices));
28828 max_free_space_items = max (MIN_NUM_FREE_SPACES, free_space_items * 3 / 2);
28829 max_free_space_items = min (MAX_NUM_FREE_SPACES, max_free_space_items);
28830 dprintf (SEG_REUSE_LOG_0, ("could fit! %Id free spaces, %Id max", free_space_items, max_free_space_items));
28836 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
28841 assert (settings.condemned_generation == (max_generation-1));
28842 size_t free_space = (end_address - heap_segment_plan_allocated (seg));
28843 size_t largest_free_space = free_space;
28844 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen1: testing segment [%Ix %Ix", first_address, end_address));
28845 //find the first free list in range of the current segment
28846 size_t sz_list = gen_allocator->first_bucket_size();
28847 unsigned int a_l_idx = 0;
28848 uint8_t* free_list = 0;
28849 for (; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
28851 if ((eph_gen_starts <= sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
28853 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
28856 if ((free_list >= first_address) &&
28857 (free_list < end_address) &&
28858 (unused_array_size (free_list) >= eph_gen_starts))
28864 free_list = free_list_slot (free_list);
28872 init_ordered_free_space_indices ();
28873 if (process_free_space (seg,
28874 unused_array_size (free_list) - eph_gen_starts + Align (min_obj_size),
28875 min_free_size, min_cont_size,
28876 &free_space, &largest_free_space))
28881 free_list = free_list_slot (free_list);
28885 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, no free list"));
28889 //tally up free space
28895 if ((free_list >= first_address) && (free_list < end_address) &&
28896 process_free_space (seg,
28897 unused_array_size (free_list),
28898 min_free_size, min_cont_size,
28899 &free_space, &largest_free_space))
28904 free_list = free_list_slot (free_list);
28907 if (a_l_idx < gen_allocator->number_of_buckets())
28909 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
28915 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
28919 BOOL can_fit = best_fit (free_space, 0, NULL);
28922 dprintf (SEG_REUSE_LOG_0, ("(gen1)Found segment %Ix to reuse with bestfit", seg));
28926 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
28934 void gc_heap::realloc_plug (size_t last_plug_size, uint8_t*& last_plug,
28935 generation* gen, uint8_t* start_address,
28936 unsigned int& active_new_gen_number,
28937 uint8_t*& last_pinned_gap, BOOL& leftp,
28940 , mark* pinned_plug_entry
28941 #endif //SHORT_PLUGS
28944 // detect generation boundaries
28945 // make sure that active_new_gen_number is not the youngest generation.
28946 // because the generation_limit wouldn't return the right thing in this case.
28949 if ((active_new_gen_number > 1) &&
28950 (last_plug >= generation_limit (active_new_gen_number)))
28952 assert (last_plug >= start_address);
28953 active_new_gen_number--;
28954 realloc_plan_generation_start (generation_of (active_new_gen_number), gen);
28955 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
28960 // detect pinned plugs
28961 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
28963 size_t entry = deque_pinned_plug();
28964 mark* m = pinned_plug_of (entry);
28966 size_t saved_pinned_len = pinned_len(m);
28967 pinned_len(m) = last_plug - last_pinned_gap;
28968 //dprintf (3,("Adjusting pinned gap: [%Ix, %Ix[", (size_t)last_pinned_gap, (size_t)last_plug));
28970 if (m->has_post_plug_info())
28972 last_plug_size += sizeof (gap_reloc_pair);
28973 dprintf (3, ("ra pinned %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
28976 last_pinned_gap = last_plug + last_plug_size;
28977 dprintf (3, ("ra found pin %Ix, len: %Ix->%Ix, last_p: %Ix, last_p_size: %Ix",
28978 pinned_plug (m), saved_pinned_len, pinned_len (m), last_plug, last_plug_size));
28981 //we are creating a generation fault. set the cards.
28983 size_t end_card = card_of (align_on_card (last_plug + last_plug_size));
28984 size_t card = card_of (last_plug);
28985 while (card != end_card)
28992 else if (last_plug >= start_address)
28994 #ifdef FEATURE_STRUCTALIGN
28995 int requiredAlignment;
28997 node_aligninfo (last_plug, requiredAlignment, pad);
28999 // from how we previously aligned the plug's destination address,
29000 // compute the actual alignment offset.
29001 uint8_t* reloc_plug = last_plug + node_relocation_distance (last_plug);
29002 ptrdiff_t alignmentOffset = ComputeStructAlignPad(reloc_plug, requiredAlignment, 0);
29003 if (!alignmentOffset)
29005 // allocate_in_expanded_heap doesn't expect alignmentOffset to be zero.
29006 alignmentOffset = requiredAlignment;
29009 //clear the alignment info because we are reallocating
29010 clear_node_aligninfo (last_plug);
29011 #else // FEATURE_STRUCTALIGN
29012 //clear the realignment flag because we are reallocating
29013 clear_node_realigned (last_plug);
29014 #endif // FEATURE_STRUCTALIGN
29015 BOOL adjacentp = FALSE;
29016 BOOL set_padding_on_saved_p = FALSE;
29020 last_plug_size += sizeof (gap_reloc_pair);
29023 assert (pinned_plug_entry != NULL);
29024 if (last_plug_size <= sizeof (plug_and_gap))
29026 set_padding_on_saved_p = TRUE;
29028 #endif //SHORT_PLUGS
29030 dprintf (3, ("ra plug %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
29034 clear_padding_in_expand (last_plug, set_padding_on_saved_p, pinned_plug_entry);
29035 #endif //SHORT_PLUGS
29037 uint8_t* new_address = allocate_in_expanded_heap(gen, last_plug_size, adjacentp, last_plug,
29039 set_padding_on_saved_p,
29041 #endif //SHORT_PLUGS
29042 TRUE, active_new_gen_number REQD_ALIGN_AND_OFFSET_ARG);
29044 dprintf (3, ("ra NA: [%Ix, %Ix[: %Ix", new_address, (new_address + last_plug_size), last_plug_size));
29045 assert (new_address);
29046 set_node_relocation_distance (last_plug, new_address - last_plug);
29047 #ifdef FEATURE_STRUCTALIGN
29048 if (leftp && node_alignpad (last_plug) == 0)
29049 #else // FEATURE_STRUCTALIGN
29050 if (leftp && !node_realigned (last_plug))
29051 #endif // FEATURE_STRUCTALIGN
29053 // TODO - temporarily disable L optimization because of a bug in it.
29054 //set_node_left (last_plug);
29056 dprintf (3,(" Re-allocating %Ix->%Ix len %Id", (size_t)last_plug, (size_t)new_address, last_plug_size));
29061 void gc_heap::realloc_in_brick (uint8_t* tree, uint8_t*& last_plug,
29062 uint8_t* start_address,
29064 unsigned int& active_new_gen_number,
29065 uint8_t*& last_pinned_gap, BOOL& leftp)
29067 assert (tree != NULL);
29068 int left_node = node_left_child (tree);
29069 int right_node = node_right_child (tree);
29071 dprintf (3, ("ra: tree: %Ix, last_pin_gap: %Ix, last_p: %Ix, L: %d, R: %d",
29072 tree, last_pinned_gap, last_plug, left_node, right_node));
29076 dprintf (3, ("LN: realloc %Ix(%Ix)", (tree + left_node), last_plug));
29077 realloc_in_brick ((tree + left_node), last_plug, start_address,
29078 gen, active_new_gen_number, last_pinned_gap,
29082 if (last_plug != 0)
29084 uint8_t* plug = tree;
29086 BOOL has_pre_plug_info_p = FALSE;
29087 BOOL has_post_plug_info_p = FALSE;
29088 mark* pinned_plug_entry = get_next_pinned_entry (tree,
29089 &has_pre_plug_info_p,
29090 &has_post_plug_info_p,
29093 // We only care about the pre plug info 'cause that's what decides if the last plug is shortened.
29094 // The pinned plugs are handled in realloc_plug.
29095 size_t gap_size = node_gap_size (plug);
29096 uint8_t* gap = (plug - gap_size);
29097 uint8_t* last_plug_end = gap;
29098 size_t last_plug_size = (last_plug_end - last_plug);
29099 // Cannot assert this - a plug could be less than that due to the shortened ones.
29100 //assert (last_plug_size >= Align (min_obj_size));
29101 dprintf (3, ("ra: plug %Ix, gap size: %Ix, last_pin_gap: %Ix, last_p: %Ix, last_p_end: %Ix, shortened: %d",
29102 plug, gap_size, last_pinned_gap, last_plug, last_plug_end, (has_pre_plug_info_p ? 1 : 0)));
29103 realloc_plug (last_plug_size, last_plug, gen, start_address,
29104 active_new_gen_number, last_pinned_gap,
29105 leftp, has_pre_plug_info_p
29107 , pinned_plug_entry
29108 #endif //SHORT_PLUGS
29116 dprintf (3, ("RN: realloc %Ix(%Ix)", (tree + right_node), last_plug));
29117 realloc_in_brick ((tree + right_node), last_plug, start_address,
29118 gen, active_new_gen_number, last_pinned_gap,
29124 gc_heap::realloc_plugs (generation* consing_gen, heap_segment* seg,
29125 uint8_t* start_address, uint8_t* end_address,
29126 unsigned active_new_gen_number)
29128 dprintf (3, ("--- Reallocing ---"));
29132 //make sure that every generation has a planned allocation start
29133 int gen_number = max_generation - 1;
29134 while (gen_number >= 0)
29136 generation* gen = generation_of (gen_number);
29137 if (0 == generation_plan_allocation_start (gen))
29139 generation_plan_allocation_start (gen) =
29140 bestfit_first_pin + (max_generation - gen_number - 1) * Align (min_obj_size);
29141 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
29142 assert (generation_plan_allocation_start (gen));
29148 uint8_t* first_address = start_address;
29149 //Look for the right pinned plug to start from.
29150 reset_pinned_queue_bos();
29151 uint8_t* planned_ephemeral_seg_end = heap_segment_plan_allocated (seg);
29152 while (!pinned_plug_que_empty_p())
29154 mark* m = oldest_pin();
29155 if ((pinned_plug (m) >= planned_ephemeral_seg_end) && (pinned_plug (m) < end_address))
29157 if (pinned_plug (m) < first_address)
29159 first_address = pinned_plug (m);
29164 deque_pinned_plug();
29167 size_t current_brick = brick_of (first_address);
29168 size_t end_brick = brick_of (end_address-1);
29169 uint8_t* last_plug = 0;
29171 uint8_t* last_pinned_gap = heap_segment_plan_allocated (seg);
29172 BOOL leftp = FALSE;
29174 dprintf (3, ("start addr: %Ix, first addr: %Ix, current oldest pin: %Ix",
29175 start_address, first_address, pinned_plug (oldest_pin())));
29177 while (current_brick <= end_brick)
29179 int brick_entry = brick_table [ current_brick ];
29180 if (brick_entry >= 0)
29182 realloc_in_brick ((brick_address (current_brick) + brick_entry - 1),
29183 last_plug, start_address, consing_gen,
29184 active_new_gen_number, last_pinned_gap,
29190 if (last_plug != 0)
29192 realloc_plug (end_address - last_plug, last_plug, consing_gen,
29194 active_new_gen_number, last_pinned_gap,
29198 #endif //SHORT_PLUGS
29202 //Fix the old segment allocated size
29203 assert (last_pinned_gap >= heap_segment_mem (seg));
29204 assert (last_pinned_gap <= heap_segment_committed (seg));
29205 heap_segment_plan_allocated (seg) = last_pinned_gap;
29208 void gc_heap::verify_no_pins (uint8_t* start, uint8_t* end)
29211 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
29213 BOOL contains_pinned_plugs = FALSE;
29216 while (mi != mark_stack_tos)
29218 m = pinned_plug_of (mi);
29219 if ((pinned_plug (m) >= start) && (pinned_plug (m) < end))
29221 contains_pinned_plugs = TRUE;
29228 if (contains_pinned_plugs)
29233 #endif //VERIFY_HEAP
29236 void gc_heap::set_expand_in_full_gc (int condemned_gen_number)
29238 if (!should_expand_in_full_gc)
29240 if ((condemned_gen_number != max_generation) &&
29241 (settings.pause_mode != pause_low_latency) &&
29242 (settings.pause_mode != pause_sustained_low_latency))
29244 should_expand_in_full_gc = TRUE;
29249 void gc_heap::save_ephemeral_generation_starts()
29251 for (int ephemeral_generation = 0; ephemeral_generation < max_generation; ephemeral_generation++)
29253 saved_ephemeral_plan_start[ephemeral_generation] =
29254 generation_plan_allocation_start (generation_of (ephemeral_generation));
29255 saved_ephemeral_plan_start_size[ephemeral_generation] =
29256 generation_plan_allocation_start_size (generation_of (ephemeral_generation));
29260 generation* gc_heap::expand_heap (int condemned_generation,
29261 generation* consing_gen,
29262 heap_segment* new_heap_segment)
29264 UNREFERENCED_PARAMETER(condemned_generation);
29265 assert (condemned_generation >= (max_generation -1));
29266 unsigned int active_new_gen_number = max_generation; //Set one too high to get generation gap
29267 uint8_t* start_address = generation_limit (max_generation);
29268 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
29269 BOOL should_promote_ephemeral = FALSE;
29270 ptrdiff_t eph_size = total_ephemeral_size;
29271 #ifdef BACKGROUND_GC
29272 dprintf(2,("%s: ---- Heap Expansion ----", (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")));
29273 #endif //BACKGROUND_GC
29274 settings.heap_expansion = TRUE;
29276 #ifdef BACKGROUND_GC
29277 if (cm_in_progress)
29279 if (!expanded_in_fgc)
29281 expanded_in_fgc = TRUE;
29284 #endif //BACKGROUND_GC
29286 //reset the elevation state for next time.
29287 dprintf (2, ("Elevation: elevation = el_none"));
29288 if (settings.should_lock_elevation && !expand_reused_seg_p())
29289 settings.should_lock_elevation = FALSE;
29291 heap_segment* new_seg = new_heap_segment;
29294 return consing_gen;
29296 //copy the card and brick tables
29297 if (g_gc_card_table!= card_table)
29298 copy_brick_card_table();
29300 BOOL new_segment_p = (heap_segment_next (new_seg) == 0);
29301 dprintf (2, ("new_segment_p %Ix", (size_t)new_segment_p));
29303 assert (generation_plan_allocation_start (generation_of (max_generation-1)));
29304 assert (generation_plan_allocation_start (generation_of (max_generation-1)) >=
29305 heap_segment_mem (ephemeral_heap_segment));
29306 assert (generation_plan_allocation_start (generation_of (max_generation-1)) <=
29307 heap_segment_committed (ephemeral_heap_segment));
29309 assert (generation_plan_allocation_start (youngest_generation));
29310 assert (generation_plan_allocation_start (youngest_generation) <
29311 heap_segment_plan_allocated (ephemeral_heap_segment));
29313 if (settings.pause_mode == pause_no_gc)
29315 // We don't reuse for no gc, so the size used on the new eph seg is eph_size.
29316 if ((size_t)(heap_segment_reserved (new_seg) - heap_segment_mem (new_seg)) < (eph_size + soh_allocation_no_gc))
29317 should_promote_ephemeral = TRUE;
29323 should_promote_ephemeral = dt_low_ephemeral_space_p (tuning_deciding_promote_ephemeral);
29327 if (should_promote_ephemeral)
29329 ephemeral_promotion = TRUE;
29330 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_new_seg_ep);
29331 dprintf (2, ("promoting ephemeral"));
29332 save_ephemeral_generation_starts();
29336 // commit the new ephemeral segment all at once if it is a new one.
29337 if ((eph_size > 0) && new_segment_p)
29339 #ifdef FEATURE_STRUCTALIGN
29340 // The destination may require a larger alignment padding than the source.
29341 // Assume the worst possible alignment padding.
29342 eph_size += ComputeStructAlignPad(heap_segment_mem (new_seg), MAX_STRUCTALIGN, OBJECT_ALIGNMENT_OFFSET);
29343 #endif // FEATURE_STRUCTALIGN
29344 #ifdef RESPECT_LARGE_ALIGNMENT
29345 //Since the generation start can be larger than min_obj_size
29346 //The alignment could be switched.
29347 eph_size += switch_alignment_size(FALSE);
29348 #endif //RESPECT_LARGE_ALIGNMENT
29349 //Since the generation start can be larger than min_obj_size
29350 //Compare the alignment of the first object in gen1
29351 if (grow_heap_segment (new_seg, heap_segment_mem (new_seg) + eph_size) == 0)
29353 fgm_result.set_fgm (fgm_commit_eph_segment, eph_size, FALSE);
29354 return consing_gen;
29356 heap_segment_used (new_seg) = heap_segment_committed (new_seg);
29359 //Fix the end of the old ephemeral heap segment
29360 heap_segment_plan_allocated (ephemeral_heap_segment) =
29361 generation_plan_allocation_start (generation_of (max_generation-1));
29363 dprintf (3, ("Old ephemeral allocated set to %Ix",
29364 (size_t)heap_segment_plan_allocated (ephemeral_heap_segment)));
29369 // TODO - Is this really necessary? We should think about it.
29370 //initialize the first brick
29371 size_t first_brick = brick_of (heap_segment_mem (new_seg));
29372 set_brick (first_brick,
29373 heap_segment_mem (new_seg) - brick_address (first_brick));
29376 //From this point on, we cannot run out of memory
29378 //reset the allocation of the consing generation back to the end of the
29379 //old ephemeral segment
29380 generation_allocation_limit (consing_gen) =
29381 heap_segment_plan_allocated (ephemeral_heap_segment);
29382 generation_allocation_pointer (consing_gen) = generation_allocation_limit (consing_gen);
29383 generation_allocation_segment (consing_gen) = ephemeral_heap_segment;
29385 //clear the generation gap for all of the ephemeral generations
29387 int generation_num = max_generation-1;
29388 while (generation_num >= 0)
29390 generation* gen = generation_of (generation_num);
29391 generation_plan_allocation_start (gen) = 0;
29396 heap_segment* old_seg = ephemeral_heap_segment;
29397 ephemeral_heap_segment = new_seg;
29399 //Note: the ephemeral segment shouldn't be threaded onto the segment chain
29400 //because the relocation and compact phases shouldn't see it
29402 // set the generation members used by allocate_in_expanded_heap
29403 // and switch to ephemeral generation
29404 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
29406 if (!should_promote_ephemeral)
29408 realloc_plugs (consing_gen, old_seg, start_address, end_address,
29409 active_new_gen_number);
29414 repair_allocation_in_expanded_heap (consing_gen);
29417 // assert that the generation gap for all of the ephemeral generations were allocated.
29420 int generation_num = max_generation-1;
29421 while (generation_num >= 0)
29423 generation* gen = generation_of (generation_num);
29424 assert (generation_plan_allocation_start (gen));
29430 if (!new_segment_p)
29432 dprintf (2, ("Demoting ephemeral segment"));
29433 //demote the entire segment.
29434 settings.demotion = TRUE;
29435 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
29436 demotion_low = heap_segment_mem (ephemeral_heap_segment);
29437 demotion_high = heap_segment_reserved (ephemeral_heap_segment);
29441 demotion_low = MAX_PTR;
29443 #ifndef MULTIPLE_HEAPS
29444 settings.demotion = FALSE;
29445 get_gc_data_per_heap()->clear_mechanism_bit (gc_demotion_bit);
29446 #endif //!MULTIPLE_HEAPS
29448 ptrdiff_t eph_size1 = total_ephemeral_size;
29449 MAYBE_UNUSED_VAR(eph_size1);
29451 if (!should_promote_ephemeral && new_segment_p)
29453 assert (eph_size1 <= eph_size);
29456 if (heap_segment_mem (old_seg) == heap_segment_plan_allocated (old_seg))
29458 // This is to catch when we accidently delete a segment that has pins.
29459 verify_no_pins (heap_segment_mem (old_seg), heap_segment_reserved (old_seg));
29462 verify_no_pins (heap_segment_plan_allocated (old_seg), heap_segment_reserved(old_seg));
29464 dprintf(2,("---- End of Heap Expansion ----"));
29465 return consing_gen;
29468 void gc_heap::set_static_data()
29470 static_data* pause_mode_sdata = static_data_table[latency_level];
29471 for (int i = 0; i < NUMBERGENERATIONS; i++)
29473 dynamic_data* dd = dynamic_data_of (i);
29474 static_data* sdata = &pause_mode_sdata[i];
29477 dd->min_size = sdata->min_size;
29479 dprintf (GTC_LOG, ("PM: %d - min: %Id, max: %Id, fr_l: %Id, fr_b: %d%%",
29480 settings.pause_mode,
29481 dd->min_size, dd_max_size,
29482 dd->fragmentation_limit, (int)(dd->fragmentation_burden_limit * 100)));
29486 // Initialize the values that are not const.
29487 void gc_heap::init_static_data()
29489 size_t gen0size = GCHeap::GetValidGen0MaxSize(get_valid_segment_size());
29490 size_t gen0_min_size = Align(gen0size / 8 * 5);
29492 size_t gen0_max_size =
29493 #ifdef MULTIPLE_HEAPS
29494 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024));
29495 #else //MULTIPLE_HEAPS
29496 (gc_can_use_concurrent ?
29498 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024)));
29499 #endif //MULTIPLE_HEAPS
29501 // TODO: gen0_max_size has a 200mb cap; gen1_max_size should also have a cap.
29502 size_t gen1_max_size =
29503 #ifdef MULTIPLE_HEAPS
29504 max (6*1024*1024, Align(soh_segment_size/2));
29505 #else //MULTIPLE_HEAPS
29506 (gc_can_use_concurrent ?
29508 max (6*1024*1024, Align(soh_segment_size/2)));
29509 #endif //MULTIPLE_HEAPS
29511 dprintf (GTC_LOG, ("gen0size: %Id, gen0 min: %Id, max: %Id, gen1 max: %Id",
29512 gen0size, gen0_min_size, gen0_max_size, gen1_max_size));
29514 for (int i = latency_level_first; i <= latency_level_last; i++)
29516 static_data_table[i][0].min_size = gen0_min_size;
29517 static_data_table[i][0].max_size = gen0_max_size;
29518 static_data_table[i][1].max_size = gen1_max_size;
29522 bool gc_heap::init_dynamic_data()
29524 qpf = GCToOSInterface::QueryPerformanceFrequency();
29526 uint32_t now = (uint32_t)GetHighPrecisionTimeStamp();
29530 for (int i = 0; i <= max_generation+1; i++)
29532 dynamic_data* dd = dynamic_data_of (i);
29534 dd->time_clock = now;
29535 dd->current_size = 0;
29536 dd->promoted_size = 0;
29537 dd->collection_count = 0;
29538 dd->new_allocation = dd->min_size;
29539 dd->gc_new_allocation = dd->new_allocation;
29540 dd->desired_allocation = dd->new_allocation;
29541 dd->fragmentation = 0;
29544 #ifdef GC_CONFIG_DRIVEN
29545 if (heap_number == 0)
29547 #endif //GC_CONFIG_DRIVEN
29552 float gc_heap::surv_to_growth (float cst, float limit, float max_limit)
29554 if (cst < ((max_limit - limit ) / (limit * (max_limit-1.0f))))
29555 return ((limit - limit*cst) / (1.0f - (cst * limit)));
29561 //if the allocation budget wasn't exhausted, the new budget may be wrong because the survival may
29562 //not be correct (collection happened too soon). Correct with a linear estimation based on the previous
29563 //value of the budget
29564 static size_t linear_allocation_model (float allocation_fraction, size_t new_allocation,
29565 size_t previous_desired_allocation, size_t collection_count)
29567 if ((allocation_fraction < 0.95) && (allocation_fraction > 0.0))
29569 dprintf (2, ("allocation fraction: %d", (int)(allocation_fraction/100.0)));
29570 new_allocation = (size_t)(allocation_fraction*new_allocation + (1.0-allocation_fraction)*previous_desired_allocation);
29573 size_t smoothing = 3; // exponential smoothing factor
29574 if (smoothing > collection_count)
29575 smoothing = collection_count;
29576 new_allocation = new_allocation / smoothing + ((previous_desired_allocation / smoothing) * (smoothing-1));
29578 UNREFERENCED_PARAMETER(collection_count);
29580 return new_allocation;
29583 size_t gc_heap::desired_new_allocation (dynamic_data* dd,
29584 size_t out, int gen_number,
29587 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
29589 if (dd_begin_data_size (dd) == 0)
29591 size_t new_allocation = dd_min_size (dd);
29592 current_gc_data_per_heap->gen_data[gen_number].new_allocation = new_allocation;
29593 return new_allocation;
29598 size_t previous_desired_allocation = dd_desired_allocation (dd);
29599 size_t current_size = dd_current_size (dd);
29600 float max_limit = dd_max_limit (dd);
29601 float limit = dd_limit (dd);
29602 size_t min_gc_size = dd_min_size (dd);
29604 size_t max_size = dd_max_size (dd);
29605 size_t new_allocation = 0;
29606 float allocation_fraction = (float) (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)) / (float) (dd_desired_allocation (dd));
29607 if (gen_number >= max_generation)
29609 size_t new_size = 0;
29611 cst = min (1.0f, float (out) / float (dd_begin_data_size (dd)));
29613 f = surv_to_growth (cst, limit, max_limit);
29614 size_t max_growth_size = (size_t)(max_size / f);
29615 if (current_size >= max_growth_size)
29617 new_size = max_size;
29621 new_size = (size_t) min (max ( (f * current_size), min_gc_size), max_size);
29624 assert ((new_size >= current_size) || (new_size == max_size));
29626 if (gen_number == max_generation)
29628 new_allocation = max((new_size - current_size), min_gc_size);
29630 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
29631 dd_desired_allocation (dd), dd_collection_count (dd));
29633 if ((dd_fragmentation (dd) > ((size_t)((f-1)*current_size))))
29635 //reducing allocation in case of fragmentation
29636 size_t new_allocation1 = max (min_gc_size,
29638 (size_t)((float)new_allocation * current_size /
29639 ((float)current_size + 2*dd_fragmentation (dd))));
29640 dprintf (2, ("Reducing max_gen allocation due to fragmentation from %Id to %Id",
29641 new_allocation, new_allocation1));
29642 new_allocation = new_allocation1;
29645 else //large object heap
29647 uint32_t memory_load = 0;
29648 uint64_t available_physical = 0;
29649 get_memory_info (&memory_load, &available_physical);
29650 if (heap_number == 0)
29651 settings.exit_memory_load = memory_load;
29652 if (available_physical > 1024*1024)
29653 available_physical -= 1024*1024;
29655 uint64_t available_free = available_physical + (uint64_t)generation_free_list_space (generation_of (gen_number));
29656 if (available_free > (uint64_t)MAX_PTR)
29658 available_free = (uint64_t)MAX_PTR;
29661 //try to avoid OOM during large object allocation
29662 new_allocation = max (min(max((new_size - current_size), dd_desired_allocation (dynamic_data_of (max_generation))),
29663 (size_t)available_free),
29664 max ((current_size/4), min_gc_size));
29666 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
29667 dd_desired_allocation (dd), dd_collection_count (dd));
29673 size_t survivors = out;
29674 cst = float (survivors) / float (dd_begin_data_size (dd));
29675 f = surv_to_growth (cst, limit, max_limit);
29676 new_allocation = (size_t) min (max ((f * (survivors)), min_gc_size), max_size);
29678 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
29679 dd_desired_allocation (dd), dd_collection_count (dd));
29681 if (gen_number == 0)
29686 //printf ("%f, %Id\n", cst, new_allocation);
29687 size_t free_space = generation_free_list_space (generation_of (gen_number));
29688 // DTREVIEW - is min_gc_size really a good choice?
29689 // on 64-bit this will almost always be true.
29690 dprintf (GTC_LOG, ("frag: %Id, min: %Id", free_space, min_gc_size));
29691 if (free_space > min_gc_size)
29693 settings.gen0_reduction_count = 2;
29697 if (settings.gen0_reduction_count > 0)
29698 settings.gen0_reduction_count--;
29701 if (settings.gen0_reduction_count > 0)
29703 dprintf (2, ("Reducing new allocation based on fragmentation"));
29704 new_allocation = min (new_allocation,
29705 max (min_gc_size, (max_size/3)));
29710 size_t new_allocation_ret =
29711 Align (new_allocation, get_alignment_constant (!(gen_number == (max_generation+1))));
29712 int gen_data_index = gen_number;
29713 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_data_index]);
29714 gen_data->new_allocation = new_allocation_ret;
29716 dd_surv (dd) = cst;
29718 #ifdef SIMPLE_DPRINTF
29719 dprintf (1, ("h%d g%d surv: %Id current: %Id alloc: %Id (%d%%) f: %d%% new-size: %Id new-alloc: %Id",
29720 heap_number, gen_number, out, current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)),
29721 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
29723 dprintf (1,("gen: %d in: %Id out: %Id ", gen_number, generation_allocation_size (generation_of (gen_number)), out));
29724 dprintf (1,("current: %Id alloc: %Id ", current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd))));
29725 dprintf (1,(" surv: %d%% f: %d%% new-size: %Id new-alloc: %Id",
29726 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
29727 #endif //SIMPLE_DPRINTF
29729 return new_allocation_ret;
29733 //returns the planned size of a generation (including free list element)
29734 size_t gc_heap::generation_plan_size (int gen_number)
29736 if (0 == gen_number)
29737 return max((heap_segment_plan_allocated (ephemeral_heap_segment) -
29738 generation_plan_allocation_start (generation_of (gen_number))),
29739 (int)Align (min_obj_size));
29742 generation* gen = generation_of (gen_number);
29743 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
29744 return (generation_plan_allocation_start (generation_of (gen_number - 1)) -
29745 generation_plan_allocation_start (generation_of (gen_number)));
29748 size_t gensize = 0;
29749 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
29751 PREFIX_ASSUME(seg != NULL);
29753 while (seg && (seg != ephemeral_heap_segment))
29755 gensize += heap_segment_plan_allocated (seg) -
29756 heap_segment_mem (seg);
29757 seg = heap_segment_next_rw (seg);
29761 gensize += (generation_plan_allocation_start (generation_of (gen_number - 1)) -
29762 heap_segment_mem (ephemeral_heap_segment));
29770 //returns the size of a generation (including free list element)
29771 size_t gc_heap::generation_size (int gen_number)
29773 if (0 == gen_number)
29774 return max((heap_segment_allocated (ephemeral_heap_segment) -
29775 generation_allocation_start (generation_of (gen_number))),
29776 (int)Align (min_obj_size));
29779 generation* gen = generation_of (gen_number);
29780 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
29781 return (generation_allocation_start (generation_of (gen_number - 1)) -
29782 generation_allocation_start (generation_of (gen_number)));
29785 size_t gensize = 0;
29786 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
29788 PREFIX_ASSUME(seg != NULL);
29790 while (seg && (seg != ephemeral_heap_segment))
29792 gensize += heap_segment_allocated (seg) -
29793 heap_segment_mem (seg);
29794 seg = heap_segment_next_rw (seg);
29798 gensize += (generation_allocation_start (generation_of (gen_number - 1)) -
29799 heap_segment_mem (ephemeral_heap_segment));
29808 size_t gc_heap::compute_in (int gen_number)
29810 assert (gen_number != 0);
29811 dynamic_data* dd = dynamic_data_of (gen_number);
29813 size_t in = generation_allocation_size (generation_of (gen_number));
29815 if (gen_number == max_generation && ephemeral_promotion)
29818 for (int i = 0; i <= max_generation; i++)
29820 dynamic_data* dd = dynamic_data_of (i);
29821 in += dd_survived_size (dd);
29822 if (i != max_generation)
29824 generation_condemned_allocated (generation_of (gen_number)) += dd_survived_size (dd);
29829 dd_gc_new_allocation (dd) -= in;
29830 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
29832 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
29833 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
29836 generation_allocation_size (generation_of (gen_number)) = 0;
29840 void gc_heap::compute_promoted_allocation (int gen_number)
29842 compute_in (gen_number);
29847 size_t gc_heap::trim_youngest_desired (uint32_t memory_load,
29848 size_t total_new_allocation,
29849 size_t total_min_allocation)
29851 if (memory_load < MAX_ALLOWED_MEM_LOAD)
29853 // If the total of memory load and gen0 budget exceeds
29854 // our max memory load limit, trim the gen0 budget so the total
29855 // is the max memory load limit.
29856 size_t remain_memory_load = (MAX_ALLOWED_MEM_LOAD - memory_load) * mem_one_percent;
29857 return min (total_new_allocation, remain_memory_load);
29861 return max (mem_one_percent, total_min_allocation);
29865 size_t gc_heap::joined_youngest_desired (size_t new_allocation)
29867 dprintf (2, ("Entry memory load: %d; gen0 new_alloc: %Id", settings.entry_memory_load, new_allocation));
29869 size_t final_new_allocation = new_allocation;
29870 if (new_allocation > MIN_YOUNGEST_GEN_DESIRED)
29872 uint32_t num_heaps = 1;
29874 #ifdef MULTIPLE_HEAPS
29875 num_heaps = gc_heap::n_heaps;
29876 #endif //MULTIPLE_HEAPS
29878 size_t total_new_allocation = new_allocation * num_heaps;
29879 size_t total_min_allocation = MIN_YOUNGEST_GEN_DESIRED * num_heaps;
29881 if ((settings.entry_memory_load >= MAX_ALLOWED_MEM_LOAD) ||
29882 (total_new_allocation > max (youngest_gen_desired_th, total_min_allocation)))
29884 uint32_t memory_load = 0;
29885 get_memory_info (&memory_load);
29886 settings.exit_memory_load = memory_load;
29887 dprintf (2, ("Current emory load: %d", memory_load));
29889 size_t final_total =
29890 trim_youngest_desired (memory_load, total_new_allocation, total_min_allocation);
29891 size_t max_new_allocation =
29892 #ifdef MULTIPLE_HEAPS
29893 dd_max_size (g_heaps[0]->dynamic_data_of (0));
29894 #else //MULTIPLE_HEAPS
29895 dd_max_size (dynamic_data_of (0));
29896 #endif //MULTIPLE_HEAPS
29898 final_new_allocation = min (Align ((final_total / num_heaps), get_alignment_constant (TRUE)), max_new_allocation);
29902 if (final_new_allocation < new_allocation)
29904 settings.gen0_reduction_count = 2;
29907 return final_new_allocation;
29912 gc_history_per_heap* gc_heap::get_gc_data_per_heap()
29914 #ifdef BACKGROUND_GC
29915 return (settings.concurrent ? &bgc_data_per_heap : &gc_data_per_heap);
29917 return &gc_data_per_heap;
29918 #endif //BACKGROUND_GC
29921 void gc_heap::compute_new_dynamic_data (int gen_number)
29923 PREFIX_ASSUME(gen_number >= 0);
29924 PREFIX_ASSUME(gen_number <= max_generation);
29926 dynamic_data* dd = dynamic_data_of (gen_number);
29927 generation* gen = generation_of (gen_number);
29928 size_t in = (gen_number==0) ? 0 : compute_in (gen_number);
29930 size_t total_gen_size = generation_size (gen_number);
29931 //keep track of fragmentation
29932 dd_fragmentation (dd) = generation_free_list_space (gen) + generation_free_obj_space (gen);
29933 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
29935 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
29937 size_t out = dd_survived_size (dd);
29939 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
29940 gen_data->size_after = total_gen_size;
29941 gen_data->free_list_space_after = generation_free_list_space (gen);
29942 gen_data->free_obj_space_after = generation_free_obj_space (gen);
29944 if ((settings.pause_mode == pause_low_latency) && (gen_number <= 1))
29946 // When we are in the low latency mode, we can still be
29947 // condemning more than gen1's 'cause of induced GCs.
29948 dd_desired_allocation (dd) = low_latency_alloc;
29952 if (gen_number == 0)
29954 //compensate for dead finalizable objects promotion.
29955 //they shoudn't be counted for growth.
29956 size_t final_promoted = 0;
29957 final_promoted = min (promoted_bytes (heap_number), out);
29958 // Prefast: this is clear from above but prefast needs to be told explicitly
29959 PREFIX_ASSUME(final_promoted <= out);
29961 dprintf (2, ("gen: %d final promoted: %Id", gen_number, final_promoted));
29962 dd_freach_previous_promotion (dd) = final_promoted;
29963 size_t lower_bound = desired_new_allocation (dd, out-final_promoted, gen_number, 0);
29965 if (settings.condemned_generation == 0)
29967 //there is no noise.
29968 dd_desired_allocation (dd) = lower_bound;
29972 size_t higher_bound = desired_new_allocation (dd, out, gen_number, 1);
29974 // <TODO>This assert was causing AppDomains\unload\test1n\test1nrun.bat to fail</TODO>
29975 //assert ( lower_bound <= higher_bound);
29977 //discount the noise. Change the desired allocation
29978 //only if the previous value is outside of the range.
29979 if (dd_desired_allocation (dd) < lower_bound)
29981 dd_desired_allocation (dd) = lower_bound;
29983 else if (dd_desired_allocation (dd) > higher_bound)
29985 dd_desired_allocation (dd) = higher_bound;
29987 #if defined (BIT64) && !defined (MULTIPLE_HEAPS)
29988 dd_desired_allocation (dd) = joined_youngest_desired (dd_desired_allocation (dd));
29989 #endif // BIT64 && !MULTIPLE_HEAPS
29990 trim_youngest_desired_low_memory();
29991 dprintf (2, ("final gen0 new_alloc: %Id", dd_desired_allocation (dd)));
29996 dd_desired_allocation (dd) = desired_new_allocation (dd, out, gen_number, 0);
30000 gen_data->pinned_surv = dd_pinned_survived_size (dd);
30001 gen_data->npinned_surv = dd_survived_size (dd) - dd_pinned_survived_size (dd);
30003 dd_gc_new_allocation (dd) = dd_desired_allocation (dd);
30004 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30007 dd_promoted_size (dd) = out;
30008 if (gen_number == max_generation)
30010 dd = dynamic_data_of (max_generation+1);
30011 total_gen_size = generation_size (max_generation + 1);
30012 dd_fragmentation (dd) = generation_free_list_space (large_object_generation) +
30013 generation_free_obj_space (large_object_generation);
30014 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
30015 dd_survived_size (dd) = dd_current_size (dd);
30017 out = dd_current_size (dd);
30018 dd_desired_allocation (dd) = desired_new_allocation (dd, out, max_generation+1, 0);
30019 dd_gc_new_allocation (dd) = Align (dd_desired_allocation (dd),
30020 get_alignment_constant (FALSE));
30021 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30023 gen_data = &(current_gc_data_per_heap->gen_data[max_generation+1]);
30024 gen_data->size_after = total_gen_size;
30025 gen_data->free_list_space_after = generation_free_list_space (large_object_generation);
30026 gen_data->free_obj_space_after = generation_free_obj_space (large_object_generation);
30027 gen_data->npinned_surv = out;
30028 #ifdef BACKGROUND_GC
30029 end_loh_size = total_gen_size;
30030 #endif //BACKGROUND_GC
30032 dd_promoted_size (dd) = out;
30036 void gc_heap::trim_youngest_desired_low_memory()
30038 if (g_low_memory_status)
30040 size_t committed_mem = 0;
30041 heap_segment* seg = generation_start_segment (generation_of (max_generation));
30044 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30045 seg = heap_segment_next (seg);
30047 seg = generation_start_segment (generation_of (max_generation + 1));
30050 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30051 seg = heap_segment_next (seg);
30054 dynamic_data* dd = dynamic_data_of (0);
30055 size_t current = dd_desired_allocation (dd);
30056 size_t candidate = max (Align ((committed_mem / 10), get_alignment_constant(FALSE)), dd_min_size (dd));
30058 dd_desired_allocation (dd) = min (current, candidate);
30062 void gc_heap::decommit_ephemeral_segment_pages()
30064 if (settings.concurrent)
30069 size_t slack_space = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30070 dynamic_data* dd = dynamic_data_of (0);
30072 #ifndef MULTIPLE_HEAPS
30073 size_t extra_space = (g_low_memory_status ? 0 : (512 * 1024));
30074 size_t decommit_timeout = (g_low_memory_status ? 0 : GC_EPHEMERAL_DECOMMIT_TIMEOUT);
30075 size_t ephemeral_elapsed = dd_time_clock(dd) - gc_last_ephemeral_decommit_time;
30077 if (dd_desired_allocation (dd) > gc_gen0_desired_high)
30079 gc_gen0_desired_high = dd_desired_allocation (dd) + extra_space;
30082 if (ephemeral_elapsed >= decommit_timeout)
30084 slack_space = min (slack_space, gc_gen0_desired_high);
30086 gc_last_ephemeral_decommit_time = dd_time_clock(dd);
30087 gc_gen0_desired_high = 0;
30089 #endif //!MULTIPLE_HEAPS
30091 if (settings.condemned_generation >= (max_generation-1))
30093 size_t new_slack_space =
30095 max(min(min(soh_segment_size/32, dd_max_size(dd)), (generation_size (max_generation) / 10)), dd_desired_allocation(dd));
30097 #ifdef FEATURE_CORECLR
30098 dd_desired_allocation (dd);
30101 #endif //FEATURE_CORECLR
30104 slack_space = min (slack_space, new_slack_space);
30107 decommit_heap_segment_pages (ephemeral_heap_segment, slack_space);
30109 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30110 current_gc_data_per_heap->extra_gen0_committed = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30113 size_t gc_heap::new_allocation_limit (size_t size, size_t free_size, int gen_number)
30115 dynamic_data* dd = dynamic_data_of (gen_number);
30116 ptrdiff_t new_alloc = dd_new_allocation (dd);
30117 assert (new_alloc == (ptrdiff_t)Align (new_alloc,
30118 get_alignment_constant (!(gen_number == (max_generation+1)))));
30119 size_t limit = min (max (new_alloc, (ptrdiff_t)size), (ptrdiff_t)free_size);
30120 assert (limit == Align (limit, get_alignment_constant (!(gen_number == (max_generation+1)))));
30121 dd_new_allocation (dd) = (new_alloc - limit );
30125 //This is meant to be called by decide_on_compacting.
30127 size_t gc_heap::generation_fragmentation (generation* gen,
30128 generation* consing_gen,
30132 uint8_t* alloc = generation_allocation_pointer (consing_gen);
30133 // If the allocation pointer has reached the ephemeral segment
30134 // fine, otherwise the whole ephemeral segment is considered
30136 if (in_range_for_segment (alloc, ephemeral_heap_segment))
30138 if (alloc <= heap_segment_allocated(ephemeral_heap_segment))
30139 frag = end - alloc;
30142 // case when no survivors, allocated set to beginning
30145 dprintf (3, ("ephemeral frag: %Id", frag));
30148 frag = (heap_segment_allocated (ephemeral_heap_segment) -
30149 heap_segment_mem (ephemeral_heap_segment));
30150 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30152 PREFIX_ASSUME(seg != NULL);
30154 while (seg != ephemeral_heap_segment)
30156 frag += (heap_segment_allocated (seg) -
30157 heap_segment_plan_allocated (seg));
30158 dprintf (3, ("seg: %Ix, frag: %Id", (size_t)seg,
30159 (heap_segment_allocated (seg) -
30160 heap_segment_plan_allocated (seg))));
30162 seg = heap_segment_next_rw (seg);
30165 dprintf (3, ("frag: %Id discounting pinned plugs", frag));
30166 //add the length of the dequeued plug free space
30168 while (bos < mark_stack_bos)
30170 frag += (pinned_len (pinned_plug_of (bos)));
30177 // for SOH this returns the total sizes of the generation and its
30178 // younger generation(s).
30179 // for LOH this returns just LOH size.
30180 size_t gc_heap::generation_sizes (generation* gen)
30183 if (generation_start_segment (gen ) == ephemeral_heap_segment)
30184 result = (heap_segment_allocated (ephemeral_heap_segment) -
30185 generation_allocation_start (gen));
30188 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
30190 PREFIX_ASSUME(seg != NULL);
30194 result += (heap_segment_allocated (seg) -
30195 heap_segment_mem (seg));
30196 seg = heap_segment_next_in_range (seg);
30203 BOOL gc_heap::decide_on_compacting (int condemned_gen_number,
30204 size_t fragmentation,
30205 BOOL& should_expand)
30207 BOOL should_compact = FALSE;
30208 should_expand = FALSE;
30209 generation* gen = generation_of (condemned_gen_number);
30210 dynamic_data* dd = dynamic_data_of (condemned_gen_number);
30211 size_t gen_sizes = generation_sizes(gen);
30212 float fragmentation_burden = ( ((0 == fragmentation) || (0 == gen_sizes)) ? (0.0f) :
30213 (float (fragmentation) / gen_sizes) );
30215 dprintf (GTC_LOG, ("fragmentation: %Id (%d%%)", fragmentation, (int)(fragmentation_burden * 100.0)));
30218 // for pure GC stress runs we need compaction, for GC stress "mix"
30219 // we need to ensure a better mix of compacting and sweeping collections
30220 if (GCStress<cfg_any>::IsEnabled() && !settings.concurrent
30221 && !g_pConfig->IsGCStressMix())
30222 should_compact = TRUE;
30225 // in GC stress "mix" mode, for stress induced collections make sure we
30226 // keep sweeps and compactions relatively balanced. do not (yet) force sweeps
30227 // against the GC's determination, as it may lead to premature OOMs.
30228 if (g_pConfig->IsGCStressMix() && settings.stress_induced)
30230 int compactions = g_GCStatistics.cntCompactFGC+g_GCStatistics.cntCompactNGC;
30231 int sweeps = g_GCStatistics.cntFGC + g_GCStatistics.cntNGC - compactions;
30232 if (compactions < sweeps / 10)
30234 should_compact = TRUE;
30238 #endif //STRESS_HEAP
30240 if (GCConfig::GetForceCompact())
30241 should_compact = TRUE;
30243 if ((condemned_gen_number == max_generation) && last_gc_before_oom)
30245 should_compact = TRUE;
30246 last_gc_before_oom = FALSE;
30247 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_last_gc);
30250 if (settings.reason == reason_induced_compacting)
30252 dprintf (2, ("induced compacting GC"));
30253 should_compact = TRUE;
30254 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_induced_compacting);
30257 dprintf (2, ("Fragmentation: %d Fragmentation burden %d%%",
30258 fragmentation, (int) (100*fragmentation_burden)));
30260 if (!should_compact)
30262 if (dt_low_ephemeral_space_p (tuning_deciding_compaction))
30264 dprintf(GTC_LOG, ("compacting due to low ephemeral"));
30265 should_compact = TRUE;
30266 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_low_ephemeral);
30270 if (should_compact)
30272 if ((condemned_gen_number >= (max_generation - 1)))
30274 if (dt_low_ephemeral_space_p (tuning_deciding_expansion))
30276 dprintf (GTC_LOG,("Not enough space for all ephemeral generations with compaction"));
30277 should_expand = TRUE;
30283 BOOL high_memory = FALSE;
30286 if (!should_compact)
30288 // We are not putting this in dt_high_frag_p because it's not exactly
30289 // high fragmentation - it's just enough planned fragmentation for us to
30290 // want to compact. Also the "fragmentation" we are talking about here
30291 // is different from anywhere else.
30292 BOOL frag_exceeded = ((fragmentation >= dd_fragmentation_limit (dd)) &&
30293 (fragmentation_burden >= dd_fragmentation_burden_limit (dd)));
30297 #ifdef BACKGROUND_GC
30298 // do not force compaction if this was a stress-induced GC
30299 IN_STRESS_HEAP(if (!settings.stress_induced))
30301 #endif // BACKGROUND_GC
30302 assert (settings.concurrent == FALSE);
30303 should_compact = TRUE;
30304 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_frag);
30305 #ifdef BACKGROUND_GC
30307 #endif // BACKGROUND_GC
30311 // check for high memory situation
30312 if(!should_compact)
30314 uint32_t num_heaps = 1;
30315 #ifdef MULTIPLE_HEAPS
30316 num_heaps = gc_heap::n_heaps;
30317 #endif // MULTIPLE_HEAPS
30319 ptrdiff_t reclaim_space = generation_size(max_generation) - generation_plan_size(max_generation);
30320 if((settings.entry_memory_load >= high_memory_load_th) && (settings.entry_memory_load < v_high_memory_load_th))
30322 if(reclaim_space > (int64_t)(min_high_fragmentation_threshold (entry_available_physical_mem, num_heaps)))
30324 dprintf(GTC_LOG,("compacting due to fragmentation in high memory"));
30325 should_compact = TRUE;
30326 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_mem_frag);
30328 high_memory = TRUE;
30330 else if(settings.entry_memory_load >= v_high_memory_load_th)
30332 if(reclaim_space > (ptrdiff_t)(min_reclaim_fragmentation_threshold (num_heaps)))
30334 dprintf(GTC_LOG,("compacting due to fragmentation in very high memory"));
30335 should_compact = TRUE;
30336 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_vhigh_mem_frag);
30338 high_memory = TRUE;
30344 // The purpose of calling ensure_gap_allocation here is to make sure
30345 // that we actually are able to commit the memory to allocate generation
30347 if ((should_compact == FALSE) &&
30348 (ensure_gap_allocation (condemned_gen_number) == FALSE))
30350 should_compact = TRUE;
30351 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_no_gaps);
30354 if (settings.condemned_generation == max_generation)
30356 //check the progress
30359 (high_memory && !should_compact) ||
30361 (generation_plan_allocation_start (generation_of (max_generation - 1)) >=
30362 generation_allocation_start (generation_of (max_generation - 1))))
30364 dprintf (2, (" Elevation: gen2 size: %d, gen2 plan size: %d, no progress, elevation = locked",
30365 generation_size (max_generation),
30366 generation_plan_size (max_generation)));
30367 //no progress -> lock
30368 settings.should_lock_elevation = TRUE;
30372 if (settings.pause_mode == pause_no_gc)
30374 should_compact = TRUE;
30375 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_plan_allocated (ephemeral_heap_segment))
30376 < soh_allocation_no_gc)
30378 should_expand = TRUE;
30382 dprintf (2, ("will %s", (should_compact ? "compact" : "sweep")));
30383 return should_compact;
30386 size_t align_lower_good_size_allocation (size_t size)
30388 return (size/64)*64;
30391 size_t gc_heap::approximate_new_allocation()
30393 dynamic_data* dd0 = dynamic_data_of (0);
30394 return max (2*dd_min_size (dd0), ((dd_desired_allocation (dd0)*2)/3));
30397 // After we did a GC we expect to have at least this
30398 // much space at the end of the segment to satisfy
30399 // a reasonable amount of allocation requests.
30400 size_t gc_heap::end_space_after_gc()
30402 return max ((dd_min_size (dynamic_data_of (0))/2), (END_SPACE_AFTER_GC + Align (min_obj_size)));
30405 BOOL gc_heap::ephemeral_gen_fit_p (gc_tuning_point tp)
30407 uint8_t* start = 0;
30409 if ((tp == tuning_deciding_condemned_gen) ||
30410 (tp == tuning_deciding_compaction))
30412 start = (settings.concurrent ? alloc_allocated : heap_segment_allocated (ephemeral_heap_segment));
30413 if (settings.concurrent)
30415 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (alloc_allocated)",
30416 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
30420 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (allocated)",
30421 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment))));
30424 else if (tp == tuning_deciding_expansion)
30426 start = heap_segment_plan_allocated (ephemeral_heap_segment);
30427 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment based on plan",
30428 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - start)));
30432 assert (tp == tuning_deciding_full_gc);
30433 dprintf (GTC_LOG, ("FGC: %Id left at the end of ephemeral segment (alloc_allocated)",
30434 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
30435 start = alloc_allocated;
30438 if (start == 0) // empty ephemeral generations
30440 assert (tp == tuning_deciding_expansion);
30441 // if there are no survivors in the ephemeral segment,
30442 // this should be the beginning of ephemeral segment.
30443 start = generation_allocation_pointer (generation_of (max_generation));
30444 assert (start == heap_segment_mem (ephemeral_heap_segment));
30447 if (tp == tuning_deciding_expansion)
30449 assert (settings.condemned_generation >= (max_generation-1));
30450 size_t gen0size = approximate_new_allocation();
30451 size_t eph_size = gen0size;
30453 for (int j = 1; j <= max_generation-1; j++)
30455 eph_size += 2*dd_min_size (dynamic_data_of(j));
30458 // We must find room for one large object and enough room for gen0size
30459 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > eph_size)
30461 dprintf (3, ("Enough room before end of segment"));
30466 size_t room = align_lower_good_size_allocation
30467 (heap_segment_reserved (ephemeral_heap_segment) - start);
30468 size_t end_seg = room;
30470 //look at the plug free space
30471 size_t largest_alloc = END_SPACE_AFTER_GC + Align (min_obj_size);
30472 bool large_chunk_found = FALSE;
30474 uint8_t* gen0start = generation_plan_allocation_start (youngest_generation);
30475 dprintf (3, ("ephemeral_gen_fit_p: gen0 plan start: %Ix", (size_t)gen0start));
30476 if (gen0start == 0)
30478 dprintf (3, ("ephemeral_gen_fit_p: room before free list search %Id, needed: %Id",
30480 while ((bos < mark_stack_bos) &&
30481 !((room >= gen0size) && large_chunk_found))
30483 uint8_t* plug = pinned_plug (pinned_plug_of (bos));
30484 if (in_range_for_segment (plug, ephemeral_heap_segment))
30486 if (plug >= gen0start)
30488 size_t chunk = align_lower_good_size_allocation (pinned_len (pinned_plug_of (bos)));
30490 if (!large_chunk_found)
30492 large_chunk_found = (chunk >= largest_alloc);
30494 dprintf (3, ("ephemeral_gen_fit_p: room now %Id, large chunk: %Id",
30495 room, large_chunk_found));
30501 if (room >= gen0size)
30503 if (large_chunk_found)
30505 dprintf (3, ("Enough room"));
30510 // now we need to find largest_alloc at the end of the segment.
30511 if (end_seg >= end_space_after_gc())
30513 dprintf (3, ("Enough room (may need end of seg)"));
30519 dprintf (3, ("Not enough room"));
30525 size_t end_space = 0;
30526 dynamic_data* dd = dynamic_data_of (0);
30527 if ((tp == tuning_deciding_condemned_gen) ||
30528 (tp == tuning_deciding_full_gc))
30530 end_space = 2*dd_min_size (dd);
30534 assert (tp == tuning_deciding_compaction);
30535 end_space = approximate_new_allocation();
30538 if (!((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > end_space))
30540 dprintf (GTC_LOG, ("ephemeral_gen_fit_p: does not fit without compaction"));
30542 return ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > end_space);
30546 CObjectHeader* gc_heap::allocate_large_object (size_t jsize, int64_t& alloc_bytes)
30548 //create a new alloc context because gen3context is shared.
30549 alloc_context acontext;
30550 acontext.alloc_ptr = 0;
30551 acontext.alloc_limit = 0;
30552 acontext.alloc_bytes = 0;
30553 #ifdef MULTIPLE_HEAPS
30554 acontext.set_alloc_heap(vm_heap);
30555 #endif //MULTIPLE_HEAPS
30558 uint8_t* current_lowest_address = lowest_address;
30559 uint8_t* current_highest_address = highest_address;
30560 #ifdef BACKGROUND_GC
30561 if (recursive_gc_sync::background_running_p())
30563 current_lowest_address = background_saved_lowest_address;
30564 current_highest_address = background_saved_highest_address;
30566 #endif //BACKGROUND_GC
30567 #endif // MARK_ARRAY
30570 size_t maxObjectSize = (INT64_MAX - 7 - Align(min_obj_size));
30572 size_t maxObjectSize = (INT32_MAX - 7 - Align(min_obj_size));
30575 if (jsize >= maxObjectSize)
30577 if (GCConfig::GetBreakOnOOM())
30579 GCToOSInterface::DebugBreak();
30584 size_t size = AlignQword (jsize);
30585 int align_const = get_alignment_constant (FALSE);
30586 #ifdef FEATURE_LOH_COMPACTION
30587 size_t pad = Align (loh_padding_obj_size, align_const);
30590 #endif //FEATURE_LOH_COMPACTION
30592 assert (size >= Align (min_obj_size, align_const));
30594 #pragma inline_depth(0)
30596 if (! allocate_more_space (&acontext, (size + pad), max_generation+1))
30602 #pragma inline_depth(20)
30605 #ifdef FEATURE_LOH_COMPACTION
30606 // The GC allocator made a free object already in this alloc context and
30607 // adjusted the alloc_ptr accordingly.
30608 #endif //FEATURE_LOH_COMPACTION
30610 uint8_t* result = acontext.alloc_ptr;
30612 assert ((size_t)(acontext.alloc_limit - acontext.alloc_ptr) == size);
30613 alloc_bytes += size;
30615 CObjectHeader* obj = (CObjectHeader*)result;
30618 if (recursive_gc_sync::background_running_p())
30620 if ((result < current_highest_address) && (result >= current_lowest_address))
30622 dprintf (3, ("Clearing mark bit at address %Ix",
30623 (size_t)(&mark_array [mark_word_of (result)])));
30625 mark_array_clear_marked (result);
30627 #ifdef BACKGROUND_GC
30628 //the object has to cover one full mark uint32_t
30629 assert (size > mark_word_size);
30630 if (current_c_gc_state == c_gc_state_marking)
30632 dprintf (3, ("Concurrent allocation of a large object %Ix",
30634 //mark the new block specially so we know it is a new object
30635 if ((result < current_highest_address) && (result >= current_lowest_address))
30637 dprintf (3, ("Setting mark bit at address %Ix",
30638 (size_t)(&mark_array [mark_word_of (result)])));
30640 mark_array_set_marked (result);
30643 #endif //BACKGROUND_GC
30645 #endif //MARK_ARRAY
30648 assert ((size_t)obj == Align ((size_t)obj, align_const));
30653 void reset_memory (uint8_t* o, size_t sizeo)
30655 if (sizeo > 128 * 1024)
30657 // We cannot reset the memory for the useful part of a free object.
30658 size_t size_to_skip = min_free_list - plug_skew;
30660 size_t page_start = align_on_page ((size_t)(o + size_to_skip));
30661 size_t size = align_lower_page ((size_t)o + sizeo - size_to_skip - plug_skew) - page_start;
30662 // Note we need to compensate for an OS bug here. This bug would cause the MEM_RESET to fail
30663 // on write watched memory.
30666 reset_mm_p = GCToOSInterface::VirtualReset((void*)page_start, size, true /* unlock */);
30671 void gc_heap::reset_large_object (uint8_t* o)
30673 // If it's a large object, allow the O/S to discard the backing store for these pages.
30674 reset_memory (o, size(o));
30677 BOOL gc_heap::large_object_marked (uint8_t* o, BOOL clearp)
30680 // It shouldn't be necessary to do these comparisons because this is only used for blocking
30681 // GCs and LOH segments cannot be out of range.
30682 if ((o >= lowest_address) && (o < highest_address))
30702 void gc_heap::walk_survivors_relocation (void* profiling_context, record_surv_fn fn)
30704 // Now walk the portion of memory that is actually being relocated.
30705 walk_relocation (profiling_context, fn);
30707 #ifdef FEATURE_LOH_COMPACTION
30708 if (loh_compacted_p)
30710 walk_relocation_for_loh (profiling_context, fn);
30712 #endif //FEATURE_LOH_COMPACTION
30715 void gc_heap::walk_survivors_for_loh (void* profiling_context, record_surv_fn fn)
30717 generation* gen = large_object_generation;
30718 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));;
30720 PREFIX_ASSUME(seg != NULL);
30722 uint8_t* o = generation_allocation_start (gen);
30723 uint8_t* plug_end = o;
30724 uint8_t* plug_start = o;
30728 if (o >= heap_segment_allocated (seg))
30730 seg = heap_segment_next (seg);
30734 o = heap_segment_mem (seg);
30736 if (large_object_marked(o, FALSE))
30743 o = o + AlignQword (size (o));
30744 if (o >= heap_segment_allocated (seg))
30748 m = large_object_marked (o, FALSE);
30753 fn (plug_start, plug_end, 0, profiling_context, false, false);
30757 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
30759 o = o + AlignQword (size (o));
30765 #ifdef BACKGROUND_GC
30767 BOOL gc_heap::background_object_marked (uint8_t* o, BOOL clearp)
30770 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
30772 if (mark_array_marked (o))
30776 mark_array_clear_marked (o);
30777 //dprintf (3, ("mark array bit for object %Ix is cleared", o));
30778 dprintf (3, ("CM: %Ix", o));
30788 dprintf (3, ("o %Ix(%d) %s", o, size(o), (m ? "was bm" : "was NOT bm")));
30792 uint8_t* gc_heap::background_next_end (heap_segment* seg, BOOL large_objects_p)
30795 (large_objects_p ? heap_segment_allocated (seg) : heap_segment_background_allocated (seg));
30798 void gc_heap::set_mem_verify (uint8_t* start, uint8_t* end, uint8_t b)
30803 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
30804 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_MEM_FILL))
30806 dprintf (3, ("setting mem to %c [%Ix, [%Ix", b, start, end));
30807 memset (start, b, (end - start));
30810 #endif //VERIFY_HEAP
30813 void gc_heap::generation_delete_heap_segment (generation* gen,
30815 heap_segment* prev_seg,
30816 heap_segment* next_seg)
30818 dprintf (3, ("bgc sweep: deleting seg %Ix", seg));
30819 if (gen == large_object_generation)
30821 heap_segment_next (prev_seg) = next_seg;
30823 dprintf (3, ("Preparing empty large segment %Ix for deletion", (size_t)seg));
30825 heap_segment_next (seg) = freeable_large_heap_segment;
30826 freeable_large_heap_segment = seg;
30830 if (seg == ephemeral_heap_segment)
30835 heap_segment_next (next_seg) = prev_seg;
30837 dprintf (3, ("Preparing empty small segment %Ix for deletion", (size_t)seg));
30838 heap_segment_next (seg) = freeable_small_heap_segment;
30839 freeable_small_heap_segment = seg;
30842 decommit_heap_segment (seg);
30843 seg->flags |= heap_segment_flags_decommitted;
30845 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
30848 void gc_heap::process_background_segment_end (heap_segment* seg,
30850 uint8_t* last_plug_end,
30851 heap_segment* start_seg,
30855 uint8_t* allocated = heap_segment_allocated (seg);
30856 uint8_t* background_allocated = heap_segment_background_allocated (seg);
30858 dprintf (3, ("Processing end of background segment [%Ix, %Ix[(%Ix[)",
30859 (size_t)heap_segment_mem (seg), background_allocated, allocated));
30862 if (allocated != background_allocated)
30864 if (gen == large_object_generation)
30869 dprintf (3, ("Make a free object before newly promoted objects [%Ix, %Ix[",
30870 (size_t)last_plug_end, background_allocated));
30871 thread_gap (last_plug_end, background_allocated - last_plug_end, generation_of (max_generation));
30873 fix_brick_to_highest (last_plug_end, background_allocated);
30875 // When we allowed fgc's during going through gaps, we could have erased the brick
30876 // that corresponds to bgc_allocated 'cause we had to update the brick there,
30877 // recover it here.
30878 fix_brick_to_highest (background_allocated, background_allocated);
30882 // by default, if allocated == background_allocated, it can't
30883 // be the ephemeral segment.
30884 if (seg == ephemeral_heap_segment)
30889 if (allocated == heap_segment_mem (seg))
30891 // this can happen with LOH segments when multiple threads
30892 // allocate new segments and not all of them were needed to
30893 // satisfy allocation requests.
30894 assert (gen == large_object_generation);
30897 if (last_plug_end == heap_segment_mem (seg))
30899 dprintf (3, ("Segment allocated is %Ix (beginning of this seg) - %s be deleted",
30900 (size_t)allocated, (*delete_p ? "should" : "should not")));
30902 if (seg != start_seg)
30909 dprintf (3, ("Trimming seg to %Ix[", (size_t)last_plug_end));
30910 heap_segment_allocated (seg) = last_plug_end;
30911 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
30913 decommit_heap_segment_pages (seg, 0);
30917 dprintf (3, ("verifying seg %Ix's mark array was completely cleared", seg));
30918 bgc_verify_mark_array_cleared (seg);
30921 void gc_heap::process_n_background_segments (heap_segment* seg,
30922 heap_segment* prev_seg,
30925 assert (gen != large_object_generation);
30929 dprintf (2, ("processing seg %Ix (not seen by bgc mark)", seg));
30930 heap_segment* next_seg = heap_segment_next (seg);
30932 if (heap_segment_read_only_p (seg))
30938 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
30940 // This can happen - if we have a LOH segment where nothing survived
30941 // or a SOH segment allocated by a gen1 GC when BGC was going where
30942 // nothing survived last time we did a gen1 GC.
30943 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
30951 verify_soh_segment_list();
30957 BOOL gc_heap::fgc_should_consider_object (uint8_t* o,
30959 BOOL consider_bgc_mark_p,
30960 BOOL check_current_sweep_p,
30961 BOOL check_saved_sweep_p)
30963 // the logic for this function must be kept in sync with the analogous function
30964 // in ToolBox\SOS\Strike\gc.cpp
30966 // TRUE means we don't need to check the bgc mark bit
30967 // FALSE means we do.
30968 BOOL no_bgc_mark_p = FALSE;
30970 if (consider_bgc_mark_p)
30972 if (check_current_sweep_p && (o < current_sweep_pos))
30974 dprintf (3, ("no bgc mark - o: %Ix < cs: %Ix", o, current_sweep_pos));
30975 no_bgc_mark_p = TRUE;
30978 if (!no_bgc_mark_p)
30980 if(check_saved_sweep_p && (o >= saved_sweep_ephemeral_start))
30982 dprintf (3, ("no bgc mark - o: %Ix >= ss: %Ix", o, saved_sweep_ephemeral_start));
30983 no_bgc_mark_p = TRUE;
30986 if (!check_saved_sweep_p)
30988 uint8_t* background_allocated = heap_segment_background_allocated (seg);
30989 // if this was the saved ephemeral segment, check_saved_sweep_p
30990 // would've been true.
30991 assert (heap_segment_background_allocated (seg) != saved_sweep_ephemeral_start);
30992 // background_allocated could be 0 for the new segments acquired during bgc
30993 // sweep and we still want no_bgc_mark_p to be true.
30994 if (o >= background_allocated)
30996 dprintf (3, ("no bgc mark - o: %Ix >= ba: %Ix", o, background_allocated));
30997 no_bgc_mark_p = TRUE;
31004 no_bgc_mark_p = TRUE;
31007 dprintf (3, ("bgc mark %Ix: %s (bm: %s)", o, (no_bgc_mark_p ? "no" : "yes"), (background_object_marked (o, FALSE) ? "yes" : "no")));
31008 return (no_bgc_mark_p ? TRUE : background_object_marked (o, FALSE));
31011 // consider_bgc_mark_p tells you if you need to care about the bgc mark bit at all
31012 // if it's TRUE, check_current_sweep_p tells you if you should consider the
31013 // current sweep position or not.
31014 void gc_heap::should_check_bgc_mark (heap_segment* seg,
31015 BOOL* consider_bgc_mark_p,
31016 BOOL* check_current_sweep_p,
31017 BOOL* check_saved_sweep_p)
31019 // the logic for this function must be kept in sync with the analogous function
31020 // in ToolBox\SOS\Strike\gc.cpp
31021 *consider_bgc_mark_p = FALSE;
31022 *check_current_sweep_p = FALSE;
31023 *check_saved_sweep_p = FALSE;
31025 if (current_c_gc_state == c_gc_state_planning)
31027 // We are doing the current_sweep_pos comparison here because we have yet to
31028 // turn on the swept flag for the segment but in_range_for_segment will return
31029 // FALSE if the address is the same as reserved.
31030 if ((seg->flags & heap_segment_flags_swept) || (current_sweep_pos == heap_segment_reserved (seg)))
31032 dprintf (3, ("seg %Ix is already swept by bgc", seg));
31036 *consider_bgc_mark_p = TRUE;
31038 dprintf (3, ("seg %Ix hasn't been swept by bgc", seg));
31040 if (seg == saved_sweep_ephemeral_seg)
31042 dprintf (3, ("seg %Ix is the saved ephemeral seg", seg));
31043 *check_saved_sweep_p = TRUE;
31046 if (in_range_for_segment (current_sweep_pos, seg))
31048 dprintf (3, ("current sweep pos is %Ix and within seg %Ix",
31049 current_sweep_pos, seg));
31050 *check_current_sweep_p = TRUE;
31056 void gc_heap::background_ephemeral_sweep()
31058 dprintf (3, ("bgc ephemeral sweep"));
31060 int align_const = get_alignment_constant (TRUE);
31062 saved_sweep_ephemeral_seg = ephemeral_heap_segment;
31063 saved_sweep_ephemeral_start = generation_allocation_start (generation_of (max_generation - 1));
31065 // Since we don't want to interfere with gen0 allocation while we are threading gen0 free list,
31066 // we thread onto a list first then publish it when we are done.
31067 allocator youngest_free_list;
31068 size_t youngest_free_list_space = 0;
31069 size_t youngest_free_obj_space = 0;
31071 youngest_free_list.clear();
31073 for (int i = 0; i <= (max_generation - 1); i++)
31075 generation* gen_to_reset = generation_of (i);
31076 assert (generation_free_list_space (gen_to_reset) == 0);
31077 // Can only assert free_list_space is 0, not free_obj_space as the allocator could have added
31078 // something there.
31081 for (int i = (max_generation - 1); i >= 0; i--)
31083 generation* current_gen = generation_of (i);
31084 uint8_t* o = generation_allocation_start (current_gen);
31085 //Skip the generation gap object
31086 o = o + Align(size (o), align_const);
31087 uint8_t* end = ((i > 0) ?
31088 generation_allocation_start (generation_of (i - 1)) :
31089 heap_segment_allocated (ephemeral_heap_segment));
31091 uint8_t* plug_end = o;
31092 uint8_t* plug_start = o;
31093 BOOL marked_p = FALSE;
31097 marked_p = background_object_marked (o, TRUE);
31101 size_t plug_size = plug_start - plug_end;
31105 thread_gap (plug_end, plug_size, current_gen);
31111 make_unused_array (plug_end, plug_size);
31112 if (plug_size >= min_free_list)
31114 youngest_free_list_space += plug_size;
31115 youngest_free_list.thread_item (plug_end, plug_size);
31119 youngest_free_obj_space += plug_size;
31124 fix_brick_to_highest (plug_end, plug_start);
31125 fix_brick_to_highest (plug_start, plug_start);
31130 o = o + Align (size (o), align_const);
31136 m = background_object_marked (o, TRUE);
31139 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31143 while ((o < end) && !background_object_marked (o, FALSE))
31145 o = o + Align (size (o), align_const);
31150 if (plug_end != end)
31154 thread_gap (plug_end, end - plug_end, current_gen);
31155 fix_brick_to_highest (plug_end, end);
31159 heap_segment_allocated (ephemeral_heap_segment) = plug_end;
31160 // the following line is temporary.
31161 heap_segment_saved_bg_allocated (ephemeral_heap_segment) = plug_end;
31163 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
31165 make_unused_array (plug_end, (end - plug_end));
31167 #endif //VERIFY_HEAP
31171 dd_fragmentation (dynamic_data_of (i)) =
31172 generation_free_list_space (current_gen) + generation_free_obj_space (current_gen);
31175 generation* youngest_gen = generation_of (0);
31176 generation_free_list_space (youngest_gen) = youngest_free_list_space;
31177 generation_free_obj_space (youngest_gen) = youngest_free_obj_space;
31178 dd_fragmentation (dynamic_data_of (0)) = youngest_free_list_space + youngest_free_obj_space;
31179 generation_allocator (youngest_gen)->copy_with_no_repair (&youngest_free_list);
31182 void gc_heap::background_sweep()
31184 generation* gen = generation_of (max_generation);
31185 dynamic_data* dd = dynamic_data_of (max_generation);
31186 // For SOH segments we go backwards.
31187 heap_segment* start_seg = ephemeral_heap_segment;
31188 PREFIX_ASSUME(start_seg != NULL);
31189 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
31190 heap_segment* seg = start_seg;
31191 uint8_t* o = heap_segment_mem (seg);
31193 heap_segment* prev_seg = heap_segment_next (seg);
31194 int align_const = get_alignment_constant (TRUE);
31197 assert (o == generation_allocation_start (generation_of (max_generation)));
31198 o = o + Align(size (o), align_const);
31201 uint8_t* plug_end = o;
31202 uint8_t* plug_start = o;
31203 next_sweep_obj = o;
31204 current_sweep_pos = o;
31206 //uint8_t* end = background_next_end (seg, (gen == large_object_generation));
31207 uint8_t* end = heap_segment_background_allocated (seg);
31208 BOOL delete_p = FALSE;
31210 //concurrent_print_time_delta ("finished with mark and start with sweep");
31211 concurrent_print_time_delta ("Sw");
31212 dprintf (2, ("---- (GC%d)Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
31214 //block concurrent allocation for large objects
31215 dprintf (3, ("lh state: planning"));
31216 if (gc_lh_block_event.IsValid())
31218 gc_lh_block_event.Reset();
31221 for (int i = 0; i <= (max_generation + 1); i++)
31223 generation* gen_to_reset = generation_of (i);
31224 generation_allocator (gen_to_reset)->clear();
31225 generation_free_list_space (gen_to_reset) = 0;
31226 generation_free_obj_space (gen_to_reset) = 0;
31227 generation_free_list_allocated (gen_to_reset) = 0;
31228 generation_end_seg_allocated (gen_to_reset) = 0;
31229 generation_condemned_allocated (gen_to_reset) = 0;
31230 //reset the allocation so foreground gc can allocate into older generation
31231 generation_allocation_pointer (gen_to_reset)= 0;
31232 generation_allocation_limit (gen_to_reset) = 0;
31233 generation_allocation_segment (gen_to_reset) = heap_segment_rw (generation_start_segment (gen_to_reset));
31236 FIRE_EVENT(BGC2ndNonConEnd);
31238 current_bgc_state = bgc_sweep_soh;
31239 verify_soh_segment_list();
31241 #ifdef FEATURE_BASICFREEZE
31242 if ((generation_start_segment (gen) != ephemeral_heap_segment) &&
31243 ro_segments_in_range)
31245 sweep_ro_segments (generation_start_segment (gen));
31247 #endif // FEATURE_BASICFREEZE
31249 //TODO BACKGROUND_GC: can we move this to where we switch to the LOH?
31250 if (current_c_gc_state != c_gc_state_planning)
31252 current_c_gc_state = c_gc_state_planning;
31255 concurrent_print_time_delta ("Swe");
31257 heap_segment* loh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation + 1)));
31258 PREFIX_ASSUME(loh_seg != NULL);
31261 loh_seg->flags &= ~heap_segment_flags_swept;
31262 heap_segment_background_allocated (loh_seg) = heap_segment_allocated (loh_seg);
31263 loh_seg = heap_segment_next_rw (loh_seg);
31266 #ifdef MULTIPLE_HEAPS
31267 bgc_t_join.join(this, gc_join_restart_ee);
31268 if (bgc_t_join.joined())
31269 #endif //MULTIPLE_HEAPS
31271 #ifdef MULTIPLE_HEAPS
31272 dprintf(2, ("Starting BGC threads for resuming EE"));
31273 bgc_t_join.restart();
31274 #endif //MULTIPLE_HEAPS
31277 if (heap_number == 0)
31282 FIRE_EVENT(BGC2ndConBegin);
31284 background_ephemeral_sweep();
31286 #ifdef MULTIPLE_HEAPS
31287 bgc_t_join.join(this, gc_join_after_ephemeral_sweep);
31288 if (bgc_t_join.joined())
31289 #endif //MULTIPLE_HEAPS
31291 #ifdef FEATURE_EVENT_TRACE
31292 bgc_heap_walk_for_etw_p = GCEventStatus::IsEnabled(GCEventProvider_Default,
31293 GCEventKeyword_GCHeapSurvivalAndMovement,
31294 GCEventLevel_Information);
31295 #endif //FEATURE_EVENT_TRACE
31297 leave_spin_lock (&gc_lock);
31299 #ifdef MULTIPLE_HEAPS
31300 dprintf(2, ("Starting BGC threads for BGC sweeping"));
31301 bgc_t_join.restart();
31302 #endif //MULTIPLE_HEAPS
31305 disable_preemptive (true);
31307 dprintf (2, ("bgs: sweeping gen2 objects"));
31308 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
31309 (size_t)heap_segment_mem (seg),
31310 (size_t)heap_segment_allocated (seg),
31311 (size_t)heap_segment_background_allocated (seg)));
31313 int num_objs = 256;
31314 int current_num_objs = 0;
31315 heap_segment* next_seg = 0;
31321 if (gen == large_object_generation)
31323 next_seg = heap_segment_next (seg);
31327 next_seg = heap_segment_prev (fseg, seg);
31332 if (!heap_segment_read_only_p (seg))
31334 if (gen == large_object_generation)
31336 // we can treat all LOH segments as in the bgc domain
31337 // regardless of whether we saw in bgc mark or not
31338 // because we don't allow LOH allocations during bgc
31339 // sweep anyway - the LOH segments can't change.
31340 process_background_segment_end (seg, gen, plug_end,
31341 start_seg, &delete_p);
31345 assert (heap_segment_background_allocated (seg) != 0);
31346 process_background_segment_end (seg, gen, plug_end,
31347 start_seg, &delete_p);
31349 assert (next_seg || !delete_p);
31355 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
31360 dprintf (2, ("seg %Ix has been swept", seg));
31361 seg->flags |= heap_segment_flags_swept;
31364 verify_soh_segment_list();
31368 dprintf (GTC_LOG, ("seg: %Ix, next_seg: %Ix, prev_seg: %Ix", seg, next_seg, prev_seg));
31372 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
31374 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
31376 if (gen != large_object_generation)
31378 dprintf (2, ("bgs: sweeping gen3 objects"));
31379 current_bgc_state = bgc_sweep_loh;
31380 gen = generation_of (max_generation+1);
31381 start_seg = heap_segment_rw (generation_start_segment (gen));
31383 PREFIX_ASSUME(start_seg != NULL);
31387 o = generation_allocation_start (gen);
31388 assert (method_table (o) == g_gc_pFreeObjectMethodTable);
31389 align_const = get_alignment_constant (FALSE);
31390 o = o + Align(size (o), align_const);
31392 end = heap_segment_allocated (seg);
31393 dprintf (2, ("sweeping gen3 objects"));
31394 generation_free_obj_space (gen) = 0;
31395 generation_allocator (gen)->clear();
31396 generation_free_list_space (gen) = 0;
31398 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
31399 (size_t)heap_segment_mem (seg),
31400 (size_t)heap_segment_allocated (seg),
31401 (size_t)heap_segment_background_allocated (seg)));
31408 o = heap_segment_mem (seg);
31411 assert (gen != large_object_generation);
31412 assert (o == generation_allocation_start (generation_of (max_generation)));
31413 align_const = get_alignment_constant (TRUE);
31414 o = o + Align(size (o), align_const);
31418 current_sweep_pos = o;
31419 next_sweep_obj = o;
31422 end = background_next_end (seg, (gen == large_object_generation));
31423 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
31424 (size_t)heap_segment_mem (seg),
31425 (size_t)heap_segment_allocated (seg),
31426 (size_t)heap_segment_background_allocated (seg)));
31430 if ((o < end) && background_object_marked (o, TRUE))
31433 if (gen == large_object_generation)
31435 dprintf (2, ("loh fr: [%Ix-%Ix[(%Id)", plug_end, plug_start, plug_start-plug_end));
31438 thread_gap (plug_end, plug_start-plug_end, gen);
31439 if (gen != large_object_generation)
31441 add_gen_free (max_generation, plug_start-plug_end);
31442 fix_brick_to_highest (plug_end, plug_start);
31443 // we need to fix the brick for the next plug here 'cause an FGC can
31444 // happen and can't read a stale brick.
31445 fix_brick_to_highest (plug_start, plug_start);
31452 next_sweep_obj = o + Align(size (o), align_const);
31453 current_num_objs++;
31454 if (current_num_objs >= num_objs)
31456 current_sweep_pos = next_sweep_obj;
31459 current_num_objs = 0;
31462 o = next_sweep_obj;
31468 m = background_object_marked (o, TRUE);
31471 if (gen != large_object_generation)
31473 add_gen_plug (max_generation, plug_end-plug_start);
31474 dd_survived_size (dd) += (plug_end - plug_start);
31476 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31480 while ((o < end) && !background_object_marked (o, FALSE))
31482 next_sweep_obj = o + Align(size (o), align_const);;
31483 current_num_objs++;
31484 if (current_num_objs >= num_objs)
31486 current_sweep_pos = plug_end;
31487 dprintf (1234, ("f: swept till %Ix", current_sweep_pos));
31489 current_num_objs = 0;
31492 o = next_sweep_obj;
31497 size_t total_loh_size = generation_size (max_generation + 1);
31498 size_t total_soh_size = generation_sizes (generation_of (max_generation));
31500 dprintf (GTC_LOG, ("h%d: S: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
31502 dprintf (GTC_LOG, ("end of bgc sweep: gen2 FL: %Id, FO: %Id",
31503 generation_free_list_space (generation_of (max_generation)),
31504 generation_free_obj_space (generation_of (max_generation))));
31505 dprintf (GTC_LOG, ("h%d: end of bgc sweep: gen3 FL: %Id, FO: %Id",
31507 generation_free_list_space (generation_of (max_generation + 1)),
31508 generation_free_obj_space (generation_of (max_generation + 1))));
31510 FIRE_EVENT(BGC2ndConEnd);
31511 concurrent_print_time_delta ("background sweep");
31513 heap_segment* reset_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
31514 PREFIX_ASSUME(reset_seg != NULL);
31518 heap_segment_saved_bg_allocated (reset_seg) = heap_segment_background_allocated (reset_seg);
31519 heap_segment_background_allocated (reset_seg) = 0;
31520 reset_seg = heap_segment_next_rw (reset_seg);
31523 // We calculate dynamic data here because if we wait till we signal the lh event,
31524 // the allocation thread can change the fragmentation and we may read an intermediate
31525 // value (which can be greater than the generation size). Plus by that time it won't
31527 compute_new_dynamic_data (max_generation);
31529 enable_preemptive ();
31531 #ifdef MULTIPLE_HEAPS
31532 bgc_t_join.join(this, gc_join_set_state_free);
31533 if (bgc_t_join.joined())
31534 #endif //MULTIPLE_HEAPS
31536 // TODO: We are using this join just to set the state. Should
31537 // look into eliminating it - check to make sure things that use
31538 // this state can live with per heap state like should_check_bgc_mark.
31539 current_c_gc_state = c_gc_state_free;
31541 #ifdef MULTIPLE_HEAPS
31542 dprintf(2, ("Starting BGC threads after background sweep phase"));
31543 bgc_t_join.restart();
31544 #endif //MULTIPLE_HEAPS
31547 disable_preemptive (true);
31549 if (gc_lh_block_event.IsValid())
31551 gc_lh_block_event.Set();
31554 //dprintf (GTC_LOG, ("---- (GC%d)End Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
31555 dprintf (GTC_LOG, ("---- (GC%d)ESw ----", VolatileLoad(&settings.gc_index)));
31557 #endif //BACKGROUND_GC
31559 void gc_heap::sweep_large_objects ()
31561 //this min value is for the sake of the dynamic tuning.
31562 //so we know that we are not starting even if we have no
31564 generation* gen = large_object_generation;
31565 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
31567 PREFIX_ASSUME(start_seg != NULL);
31569 heap_segment* seg = start_seg;
31570 heap_segment* prev_seg = 0;
31571 uint8_t* o = generation_allocation_start (gen);
31572 int align_const = get_alignment_constant (FALSE);
31574 //Skip the generation gap object
31575 o = o + Align(size (o), align_const);
31577 uint8_t* plug_end = o;
31578 uint8_t* plug_start = o;
31580 generation_allocator (gen)->clear();
31581 generation_free_list_space (gen) = 0;
31582 generation_free_obj_space (gen) = 0;
31585 dprintf (3, ("sweeping large objects"));
31586 dprintf (3, ("seg: %Ix, [%Ix, %Ix[, starting from %Ix",
31588 (size_t)heap_segment_mem (seg),
31589 (size_t)heap_segment_allocated (seg),
31594 if (o >= heap_segment_allocated (seg))
31596 heap_segment* next_seg = heap_segment_next (seg);
31597 //delete the empty segment if not the only one
31598 if ((plug_end == heap_segment_mem (seg)) &&
31599 (seg != start_seg) && !heap_segment_read_only_p (seg))
31601 //prepare for deletion
31602 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
31604 heap_segment_next (prev_seg) = next_seg;
31605 heap_segment_next (seg) = freeable_large_heap_segment;
31606 freeable_large_heap_segment = seg;
31610 if (!heap_segment_read_only_p (seg))
31612 dprintf (3, ("Trimming seg to %Ix[", (size_t)plug_end));
31613 heap_segment_allocated (seg) = plug_end;
31614 decommit_heap_segment_pages (seg, 0);
31623 o = heap_segment_mem (seg);
31625 dprintf (3, ("seg: %Ix, [%Ix, %Ix[", (size_t)seg,
31626 (size_t)heap_segment_mem (seg),
31627 (size_t)heap_segment_allocated (seg)));
31630 if (large_object_marked(o, TRUE))
31633 //everything between plug_end and plug_start is free
31634 thread_gap (plug_end, plug_start-plug_end, gen);
31639 o = o + AlignQword (size (o));
31640 if (o >= heap_segment_allocated (seg))
31644 m = large_object_marked (o, TRUE);
31647 dprintf (3, ("plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31651 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
31653 o = o + AlignQword (size (o));
31658 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
31660 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
31663 void gc_heap::relocate_in_large_objects ()
31665 relocate_args args;
31667 args.high = gc_high;
31668 args.last_plug = 0;
31670 generation* gen = large_object_generation;
31672 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
31674 PREFIX_ASSUME(seg != NULL);
31676 uint8_t* o = generation_allocation_start (gen);
31680 if (o >= heap_segment_allocated (seg))
31682 seg = heap_segment_next_rw (seg);
31687 o = heap_segment_mem (seg);
31690 while (o < heap_segment_allocated (seg))
31692 check_class_object_demotion (o);
31693 if (contain_pointers (o))
31695 dprintf(3, ("Relocating through large object %Ix", (size_t)o));
31696 go_through_object_nostart (method_table (o), o, size(o), pval,
31698 reloc_survivor_helper (pval);
31701 o = o + AlignQword (size (o));
31706 void gc_heap::mark_through_cards_for_large_objects (card_fn fn,
31709 uint8_t* low = gc_low;
31710 size_t end_card = 0;
31711 generation* oldest_gen = generation_of (max_generation+1);
31712 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
31714 PREFIX_ASSUME(seg != NULL);
31716 uint8_t* beg = generation_allocation_start (oldest_gen);
31717 uint8_t* end = heap_segment_allocated (seg);
31719 size_t cg_pointers_found = 0;
31721 size_t card_word_end = (card_of (align_on_card_word (end)) /
31726 size_t n_card_set = 0;
31727 uint8_t* next_boundary = (relocating ?
31728 generation_plan_allocation_start (generation_of (max_generation -1)) :
31731 uint8_t* nhigh = (relocating ?
31732 heap_segment_plan_allocated (ephemeral_heap_segment) :
31735 BOOL foundp = FALSE;
31736 uint8_t* start_address = 0;
31737 uint8_t* limit = 0;
31738 size_t card = card_of (beg);
31740 #ifdef BACKGROUND_GC
31741 BOOL consider_bgc_mark_p = FALSE;
31742 BOOL check_current_sweep_p = FALSE;
31743 BOOL check_saved_sweep_p = FALSE;
31744 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
31745 #endif //BACKGROUND_GC
31747 size_t total_cards_cleared = 0;
31749 //dprintf(3,( "scanning large objects from %Ix to %Ix", (size_t)beg, (size_t)end));
31750 dprintf(3, ("CMl: %Ix->%Ix", (size_t)beg, (size_t)end));
31753 if ((o < end) && (card_of(o) > card))
31755 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
31756 if (cg_pointers_found == 0)
31758 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)o));
31759 clear_cards (card, card_of((uint8_t*)o));
31760 total_cards_cleared += (card_of((uint8_t*)o) - card);
31762 n_eph +=cg_pointers_found;
31763 cg_pointers_found = 0;
31764 card = card_of ((uint8_t*)o);
31766 if ((o < end) &&(card >= end_card))
31768 foundp = find_card (card_table, card, card_word_end, end_card);
31771 n_card_set+= end_card - card;
31772 start_address = max (beg, card_address (card));
31774 limit = min (end, card_address (end_card));
31776 if ((!foundp) || (o >= end) || (card_address (card) >= end))
31778 if ((foundp) && (cg_pointers_found == 0))
31780 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
31781 (size_t)card_address(card+1)));
31782 clear_cards (card, card+1);
31783 total_cards_cleared += 1;
31785 n_eph +=cg_pointers_found;
31786 cg_pointers_found = 0;
31787 if ((seg = heap_segment_next_rw (seg)) != 0)
31789 #ifdef BACKGROUND_GC
31790 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
31791 #endif //BACKGROUND_GC
31792 beg = heap_segment_mem (seg);
31793 end = compute_next_end (seg, low);
31794 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
31795 card = card_of (beg);
31806 assert (card_set_p (card));
31808 dprintf(3,("card %Ix: o: %Ix, l: %Ix[ ",
31809 card, (size_t)o, (size_t)limit));
31811 assert (Align (size (o)) >= Align (min_obj_size));
31812 size_t s = size (o);
31813 uint8_t* next_o = o + AlignQword (s);
31819 assert (Align (s) >= Align (min_obj_size));
31820 next_o = o + AlignQword (s);
31823 dprintf (4, ("|%Ix|", (size_t)o));
31824 if (next_o < start_address)
31829 #ifdef BACKGROUND_GC
31830 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
31834 #endif //BACKGROUND_GC
31836 #ifdef COLLECTIBLE_CLASS
31837 if (is_collectible(o))
31839 BOOL passed_end_card_p = FALSE;
31841 if (card_of (o) > card)
31843 passed_end_card_p = card_transition (o, end, card_word_end,
31847 foundp, start_address,
31848 limit, total_cards_cleared);
31851 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
31853 // card is valid and it covers the head of the object
31854 if (fn == &gc_heap::relocate_address)
31856 keep_card_live (o, n_gen, cg_pointers_found);
31860 uint8_t* class_obj = get_class_object (o);
31861 mark_through_cards_helper (&class_obj, n_gen,
31862 cg_pointers_found, fn,
31863 nhigh, next_boundary);
31867 if (passed_end_card_p)
31869 if (foundp && (card_address (card) < next_o))
31871 goto go_through_refs;
31881 #endif //COLLECTIBLE_CLASS
31883 if (contain_pointers (o))
31885 dprintf(3,("Going through %Ix", (size_t)o));
31887 go_through_object (method_table(o), o, s, poo,
31888 start_address, use_start, (o + s),
31890 if (card_of ((uint8_t*)poo) > card)
31892 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
31897 foundp, start_address,
31898 limit, total_cards_cleared);
31900 if (passed_end_card_p)
31902 if (foundp && (card_address (card) < next_o))
31906 if (ppstop <= (uint8_t**)start_address)
31908 else if (poo < (uint8_t**)start_address)
31909 {poo = (uint8_t**)start_address;}
31919 mark_through_cards_helper (poo, n_gen,
31920 cg_pointers_found, fn,
31921 nhigh, next_boundary);
31933 // compute the efficiency ratio of the card table
31936 generation_skip_ratio = min (((n_eph > 800) ?
31937 (int)(((float)n_gen / (float)n_eph) * 100) : 100),
31938 generation_skip_ratio);
31940 dprintf (3, ("Mloh: cross: %Id, useful: %Id, cards cleared: %Id, cards set: %Id, ratio: %d",
31941 n_eph, n_gen, total_cards_cleared, n_card_set, generation_skip_ratio));
31945 dprintf (3, ("R: Mloh: cross: %Id, useful: %Id, cards set: %Id, ratio: %d",
31946 n_eph, n_gen, n_card_set, generation_skip_ratio));
31950 void gc_heap::descr_segment (heap_segment* seg )
31953 uint8_t* x = heap_segment_mem (seg);
31954 while (x < heap_segment_allocated (seg))
31956 dprintf(2, ( "%Ix: %d ", (size_t)x, size (x)));
31957 x = x + Align(size (x));
31960 UNREFERENCED_PARAMETER(seg);
31964 void gc_heap::descr_generations_to_profiler (gen_walk_fn fn, void *context)
31966 #ifdef MULTIPLE_HEAPS
31967 int n_heaps = g_theGCHeap->GetNumberOfHeaps ();
31968 for (int i = 0; i < n_heaps; i++)
31970 gc_heap* hp = GCHeap::GetHeap(i)->pGenGCHeap;
31971 #else //MULTIPLE_HEAPS
31973 gc_heap* hp = NULL;
31975 // prefix complains about us dereferencing hp in wks build even though we only access static members
31976 // this way. not sure how to shut it up except for this ugly workaround:
31977 PREFIX_ASSUME(hp != NULL);
31978 #endif // _PREFAST_
31979 #endif //MULTIPLE_HEAPS
31981 int curr_gen_number0 = max_generation+1;
31982 while (curr_gen_number0 >= 0)
31984 generation* gen = hp->generation_of (curr_gen_number0);
31985 heap_segment* seg = generation_start_segment (gen);
31986 while (seg && (seg != hp->ephemeral_heap_segment))
31988 assert (curr_gen_number0 > 0);
31990 // report bounds from heap_segment_mem (seg) to
31991 // heap_segment_allocated (seg);
31992 // for generation # curr_gen_number0
31993 // for heap # heap_no
31995 fn(context, curr_gen_number0, heap_segment_mem (seg),
31996 heap_segment_allocated (seg),
31997 curr_gen_number0 == max_generation+1 ? heap_segment_reserved (seg) : heap_segment_allocated (seg));
31999 seg = heap_segment_next (seg);
32003 assert (seg == hp->ephemeral_heap_segment);
32004 assert (curr_gen_number0 <= max_generation);
32006 if (curr_gen_number0 == max_generation)
32008 if (heap_segment_mem (seg) < generation_allocation_start (hp->generation_of (max_generation-1)))
32010 // report bounds from heap_segment_mem (seg) to
32011 // generation_allocation_start (generation_of (max_generation-1))
32012 // for heap # heap_number
32014 fn(context, curr_gen_number0, heap_segment_mem (seg),
32015 generation_allocation_start (hp->generation_of (max_generation-1)),
32016 generation_allocation_start (hp->generation_of (max_generation-1)) );
32019 else if (curr_gen_number0 != 0)
32021 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32022 // to generation_allocation_start (generation_of (curr_gen_number0-1))
32023 // for heap # heap_number
32025 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32026 generation_allocation_start (hp->generation_of (curr_gen_number0-1)),
32027 generation_allocation_start (hp->generation_of (curr_gen_number0-1)));
32031 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32032 // to heap_segment_allocated (ephemeral_heap_segment);
32033 // for heap # heap_number
32035 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32036 heap_segment_allocated (hp->ephemeral_heap_segment),
32037 heap_segment_reserved (hp->ephemeral_heap_segment) );
32040 curr_gen_number0--;
32046 // Note that when logging is on it can take a long time to go through the free items.
32047 void gc_heap::print_free_list (int gen, heap_segment* seg)
32049 UNREFERENCED_PARAMETER(gen);
32050 UNREFERENCED_PARAMETER(seg);
32052 if (settings.concurrent == FALSE)
32054 uint8_t* seg_start = heap_segment_mem (seg);
32055 uint8_t* seg_end = heap_segment_allocated (seg);
32057 dprintf (3, ("Free list in seg %Ix:", seg_start));
32059 size_t total_free_item = 0;
32061 allocator* gen_allocator = generation_allocator (generation_of (gen));
32062 for (unsigned int b = 0; b < gen_allocator->number_of_buckets(); b++)
32064 uint8_t* fo = gen_allocator->alloc_list_head_of (b);
32067 if (fo >= seg_start && fo < seg_end)
32071 size_t free_item_len = size(fo);
32073 dprintf (3, ("[%Ix, %Ix[:%Id",
32075 (size_t)(fo + free_item_len),
32079 fo = free_list_slot (fo);
32083 dprintf (3, ("total %Id free items", total_free_item));
32089 void gc_heap::descr_generations (BOOL begin_gc_p)
32091 UNREFERENCED_PARAMETER(begin_gc_p);
32093 if (StressLog::StressLogOn(LF_GC, LL_INFO10))
32096 #ifdef MULTIPLE_HEAPS
32098 #endif //MULTIPLE_HEAPS
32100 STRESS_LOG1(LF_GC, LL_INFO10, "GC Heap %p\n", hp);
32101 for (int n = max_generation; n >= 0; --n)
32103 STRESS_LOG4(LF_GC, LL_INFO10, " Generation %d [%p, %p] cur = %p\n",
32105 generation_allocation_start(generation_of(n)),
32106 generation_allocation_limit(generation_of(n)),
32107 generation_allocation_pointer(generation_of(n)));
32109 heap_segment* seg = generation_start_segment(generation_of(n));
32112 STRESS_LOG4(LF_GC, LL_INFO10, " Segment mem %p alloc = %p used %p committed %p\n",
32113 heap_segment_mem(seg),
32114 heap_segment_allocated(seg),
32115 heap_segment_used(seg),
32116 heap_segment_committed(seg));
32117 seg = heap_segment_next(seg);
32121 #endif // STRESS_LOG
32124 dprintf (2, ("lowest_address: %Ix highest_address: %Ix",
32125 (size_t) lowest_address, (size_t) highest_address));
32126 #ifdef BACKGROUND_GC
32127 dprintf (2, ("bgc lowest_address: %Ix bgc highest_address: %Ix",
32128 (size_t) background_saved_lowest_address, (size_t) background_saved_highest_address));
32129 #endif //BACKGROUND_GC
32131 if (heap_number == 0)
32133 dprintf (1, ("total heap size: %Id, commit size: %Id", get_total_heap_size(), get_total_committed_size()));
32136 int curr_gen_number = max_generation+1;
32137 while (curr_gen_number >= 0)
32139 size_t total_gen_size = generation_size (curr_gen_number);
32140 #ifdef SIMPLE_DPRINTF
32141 dprintf (GTC_LOG, ("[%s][g%d]gen %d:, size: %Id, frag: %Id(L: %Id, O: %Id), f: %d%% %s %s %s",
32142 (begin_gc_p ? "BEG" : "END"),
32143 settings.condemned_generation,
32146 dd_fragmentation (dynamic_data_of (curr_gen_number)),
32147 generation_free_list_space (generation_of (curr_gen_number)),
32148 generation_free_obj_space (generation_of (curr_gen_number)),
32150 (int)(((double)dd_fragmentation (dynamic_data_of (curr_gen_number)) / (double)total_gen_size) * 100) :
32152 (begin_gc_p ? ("") : (settings.compaction ? "(compact)" : "(sweep)")),
32153 (settings.heap_expansion ? "(EX)" : " "),
32154 (settings.promotion ? "Promotion" : "NoPromotion")));
32156 dprintf (2, ( "Generation %d: gap size: %d, generation size: %Id, fragmentation: %Id",
32158 size (generation_allocation_start (generation_of (curr_gen_number))),
32160 dd_fragmentation (dynamic_data_of (curr_gen_number))));
32161 #endif //SIMPLE_DPRINTF
32163 generation* gen = generation_of (curr_gen_number);
32164 heap_segment* seg = generation_start_segment (gen);
32165 while (seg && (seg != ephemeral_heap_segment))
32167 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[-%Ix[ (%Id) (%Id)",
32169 (size_t)heap_segment_mem (seg),
32170 (size_t)heap_segment_allocated (seg),
32171 (size_t)heap_segment_committed (seg),
32172 (size_t)(heap_segment_allocated (seg) - heap_segment_mem (seg)),
32173 (size_t)(heap_segment_committed (seg) - heap_segment_allocated (seg))));
32174 print_free_list (curr_gen_number, seg);
32175 seg = heap_segment_next (seg);
32177 if (seg && (seg != generation_start_segment (gen)))
32179 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32181 (size_t)heap_segment_mem (seg),
32182 (size_t)generation_allocation_start (generation_of (curr_gen_number-1))));
32183 print_free_list (curr_gen_number, seg);
32188 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32190 (size_t)generation_allocation_start (generation_of (curr_gen_number)),
32191 (size_t)(((curr_gen_number == 0)) ?
32192 (heap_segment_allocated
32193 (generation_start_segment
32194 (generation_of (curr_gen_number)))) :
32195 (generation_allocation_start
32196 (generation_of (curr_gen_number - 1))))
32198 print_free_list (curr_gen_number, seg);
32210 //-----------------------------------------------------------------------------
32212 // VM Specific support
32214 //-----------------------------------------------------------------------------
32219 unsigned int PromotedObjectCount = 0;
32220 unsigned int CreatedObjectCount = 0;
32221 unsigned int AllocDuration = 0;
32222 unsigned int AllocCount = 0;
32223 unsigned int AllocBigCount = 0;
32224 unsigned int AllocSmallCount = 0;
32225 unsigned int AllocStart = 0;
32228 //Static member variables.
32229 VOLATILE(BOOL) GCHeap::GcInProgress = FALSE;
32231 //CMCSafeLock* GCHeap::fGcLock;
32232 GCEvent *GCHeap::WaitForGCEvent = NULL;
32235 unsigned int GCHeap::GcDuration;
32237 unsigned GCHeap::GcCondemnedGeneration = 0;
32238 size_t GCHeap::totalSurvivedSize = 0;
32239 #ifdef FEATURE_PREMORTEM_FINALIZATION
32240 CFinalize* GCHeap::m_Finalize = 0;
32241 BOOL GCHeap::GcCollectClasses = FALSE;
32242 VOLATILE(int32_t) GCHeap::m_GCFLock = 0;
32244 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
32246 #ifdef BACKGROUND_GC
32247 int GCHeap::gc_stress_fgcs_in_bgc = 0;
32248 #endif // BACKGROUND_GC
32249 #ifndef MULTIPLE_HEAPS
32250 OBJECTHANDLE GCHeap::m_StressObjs[NUM_HEAP_STRESS_OBJS];
32251 int GCHeap::m_CurStressObj = 0;
32252 #endif // !MULTIPLE_HEAPS
32253 #endif // STRESS_HEAP
32254 #endif // FEATURE_REDHAWK
32256 #endif //FEATURE_PREMORTEM_FINALIZATION
32258 class NoGCRegionLockHolder
32261 NoGCRegionLockHolder()
32263 enter_spin_lock_noinstru(&g_no_gc_lock);
32266 ~NoGCRegionLockHolder()
32268 leave_spin_lock_noinstru(&g_no_gc_lock);
32272 // An explanation of locking for finalization:
32274 // Multiple threads allocate objects. During the allocation, they are serialized by
32275 // the AllocLock above. But they release that lock before they register the object
32276 // for finalization. That's because there is much contention for the alloc lock, but
32277 // finalization is presumed to be a rare case.
32279 // So registering an object for finalization must be protected by the FinalizeLock.
32281 // There is another logical queue that involves finalization. When objects registered
32282 // for finalization become unreachable, they are moved from the "registered" queue to
32283 // the "unreachable" queue. Note that this only happens inside a GC, so no other
32284 // threads can be manipulating either queue at that time. Once the GC is over and
32285 // threads are resumed, the Finalizer thread will dequeue objects from the "unreachable"
32286 // queue and call their finalizers. This dequeue operation is also protected with
32287 // the finalize lock.
32289 // At first, this seems unnecessary. Only one thread is ever enqueuing or dequeuing
32290 // on the unreachable queue (either the GC thread during a GC or the finalizer thread
32291 // when a GC is not in progress). The reason we share a lock with threads enqueuing
32292 // on the "registered" queue is that the "registered" and "unreachable" queues are
32295 // They are actually two regions of a longer list, which can only grow at one end.
32296 // So to enqueue an object to the "registered" list, you actually rotate an unreachable
32297 // object at the boundary between the logical queues, out to the other end of the
32298 // unreachable queue -- where all growing takes place. Then you move the boundary
32299 // pointer so that the gap we created at the boundary is now on the "registered"
32300 // side rather than the "unreachable" side. Now the object can be placed into the
32301 // "registered" side at that point. This is much more efficient than doing moves
32302 // of arbitrarily long regions, but it causes the two queues to require a shared lock.
32304 // Notice that Enter/LeaveFinalizeLock is not a GC-aware spin lock. Instead, it relies
32305 // on the fact that the lock will only be taken for a brief period and that it will
32306 // never provoke or allow a GC while the lock is held. This is critical. If the
32307 // FinalizeLock used enter_spin_lock (and thus sometimes enters preemptive mode to
32308 // allow a GC), then the Alloc client would have to GC protect a finalizable object
32309 // to protect against that eventuality. That is too slow!
32313 BOOL IsValidObject99(uint8_t *pObject)
32316 if (!((CObjectHeader*)pObject)->IsFree())
32317 ((CObjectHeader *) pObject)->Validate();
32318 #endif //VERIFY_HEAP
32322 #ifdef BACKGROUND_GC
32323 BOOL gc_heap::bgc_mark_array_range (heap_segment* seg,
32325 uint8_t** range_beg,
32326 uint8_t** range_end)
32328 uint8_t* seg_start = heap_segment_mem (seg);
32329 uint8_t* seg_end = (whole_seg_p ? heap_segment_reserved (seg) : align_on_mark_word (heap_segment_allocated (seg)));
32331 if ((seg_start < background_saved_highest_address) &&
32332 (seg_end > background_saved_lowest_address))
32334 *range_beg = max (seg_start, background_saved_lowest_address);
32335 *range_end = min (seg_end, background_saved_highest_address);
32344 void gc_heap::bgc_verify_mark_array_cleared (heap_segment* seg)
32346 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32347 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32349 uint8_t* range_beg = 0;
32350 uint8_t* range_end = 0;
32352 if (bgc_mark_array_range (seg, TRUE, &range_beg, &range_end))
32354 size_t markw = mark_word_of (range_beg);
32355 size_t markw_end = mark_word_of (range_end);
32356 while (markw < markw_end)
32358 if (mark_array [markw])
32360 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
32361 markw, mark_array [markw], mark_word_address (markw)));
32366 uint8_t* p = mark_word_address (markw_end);
32367 while (p < range_end)
32369 assert (!(mark_array_marked (p)));
32374 #endif //VERIFY_HEAP && MARK_ARRAY
32377 void gc_heap::verify_mark_bits_cleared (uint8_t* obj, size_t s)
32379 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32380 size_t start_mark_bit = mark_bit_of (obj) + 1;
32381 size_t end_mark_bit = mark_bit_of (obj + s);
32382 unsigned int startbit = mark_bit_bit (start_mark_bit);
32383 unsigned int endbit = mark_bit_bit (end_mark_bit);
32384 size_t startwrd = mark_bit_word (start_mark_bit);
32385 size_t endwrd = mark_bit_word (end_mark_bit);
32386 unsigned int result = 0;
32388 unsigned int firstwrd = ~(lowbits (~0, startbit));
32389 unsigned int lastwrd = ~(highbits (~0, endbit));
32391 if (startwrd == endwrd)
32393 unsigned int wrd = firstwrd & lastwrd;
32394 result = mark_array[startwrd] & wrd;
32402 // verify the first mark word is cleared.
32405 result = mark_array[startwrd] & firstwrd;
32413 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
32415 result = mark_array[wrdtmp];
32422 // set the last mark word.
32425 result = mark_array[endwrd] & lastwrd;
32431 #endif //VERIFY_HEAP && MARK_ARRAY
32434 void gc_heap::clear_all_mark_array()
32437 //size_t num_dwords_written = 0;
32438 //size_t begin_time = GetHighPrecisionTimeStamp();
32440 generation* gen = generation_of (max_generation);
32441 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32447 if (gen != large_object_generation)
32449 gen = generation_of (max_generation+1);
32450 seg = heap_segment_rw (generation_start_segment (gen));
32458 uint8_t* range_beg = 0;
32459 uint8_t* range_end = 0;
32461 if (bgc_mark_array_range (seg, (seg == ephemeral_heap_segment), &range_beg, &range_end))
32463 size_t markw = mark_word_of (range_beg);
32464 size_t markw_end = mark_word_of (range_end);
32465 size_t size_total = (markw_end - markw) * sizeof (uint32_t);
32466 //num_dwords_written = markw_end - markw;
32468 size_t size_left = 0;
32470 assert (((size_t)&mark_array[markw] & (sizeof(PTR_PTR)-1)) == 0);
32472 if ((size_total & (sizeof(PTR_PTR) - 1)) != 0)
32474 size = (size_total & ~(sizeof(PTR_PTR) - 1));
32475 size_left = size_total - size;
32476 assert ((size_left & (sizeof (uint32_t) - 1)) == 0);
32483 memclr ((uint8_t*)&mark_array[markw], size);
32485 if (size_left != 0)
32487 uint32_t* markw_to_clear = &mark_array[markw + size / sizeof (uint32_t)];
32488 for (size_t i = 0; i < (size_left / sizeof (uint32_t)); i++)
32490 *markw_to_clear = 0;
32496 seg = heap_segment_next_rw (seg);
32499 //size_t end_time = GetHighPrecisionTimeStamp() - begin_time;
32501 //printf ("took %Id ms to clear %Id bytes\n", end_time, num_dwords_written*sizeof(uint32_t));
32503 #endif //MARK_ARRAY
32506 #endif //BACKGROUND_GC
32508 void gc_heap::verify_mark_array_cleared (heap_segment* seg)
32510 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32511 assert (card_table == g_gc_card_table);
32512 size_t markw = mark_word_of (heap_segment_mem (seg));
32513 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
32515 while (markw < markw_end)
32517 if (mark_array [markw])
32519 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
32520 markw, mark_array [markw], mark_word_address (markw)));
32525 #endif //VERIFY_HEAP && MARK_ARRAY
32528 void gc_heap::verify_mark_array_cleared ()
32530 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32531 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32533 generation* gen = generation_of (max_generation);
32534 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32540 if (gen != large_object_generation)
32542 gen = generation_of (max_generation+1);
32543 seg = heap_segment_rw (generation_start_segment (gen));
32551 bgc_verify_mark_array_cleared (seg);
32552 seg = heap_segment_next_rw (seg);
32555 #endif //VERIFY_HEAP && MARK_ARRAY
32558 void gc_heap::verify_seg_end_mark_array_cleared()
32560 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32561 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32563 generation* gen = generation_of (max_generation);
32564 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32570 if (gen != large_object_generation)
32572 gen = generation_of (max_generation+1);
32573 seg = heap_segment_rw (generation_start_segment (gen));
32581 // We already cleared all mark array bits for ephemeral generations
32582 // at the beginning of bgc sweep
32583 uint8_t* from = ((seg == ephemeral_heap_segment) ?
32584 generation_allocation_start (generation_of (max_generation - 1)) :
32585 heap_segment_allocated (seg));
32586 size_t markw = mark_word_of (align_on_mark_word (from));
32587 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
32589 while (from < mark_word_address (markw))
32591 if (is_mark_bit_set (from))
32593 dprintf (3, ("mark bit for %Ix was not cleared", from));
32597 from += mark_bit_pitch;
32600 while (markw < markw_end)
32602 if (mark_array [markw])
32604 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
32605 markw, mark_array [markw], mark_word_address (markw)));
32610 seg = heap_segment_next_rw (seg);
32613 #endif //VERIFY_HEAP && MARK_ARRAY
32616 // This function is called to make sure we don't mess up the segment list
32617 // in SOH. It's called by:
32618 // 1) begin and end of ephemeral GCs
32619 // 2) during bgc sweep when we switch segments.
32620 void gc_heap::verify_soh_segment_list()
32623 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32625 generation* gen = generation_of (max_generation);
32626 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32627 heap_segment* last_seg = 0;
32631 seg = heap_segment_next_rw (seg);
32633 if (last_seg != ephemeral_heap_segment)
32638 #endif //VERIFY_HEAP
32641 // This function can be called at any foreground GCs or blocking GCs. For background GCs,
32642 // it can be called at the end of the final marking; and at any point during background
32644 // NOTE - to be able to call this function during background sweep, we need to temporarily
32645 // NOT clear the mark array bits as we go.
32646 void gc_heap::verify_partial ()
32648 #ifdef BACKGROUND_GC
32649 //printf ("GC#%d: Verifying loh during sweep\n", settings.gc_index);
32650 //generation* gen = large_object_generation;
32651 generation* gen = generation_of (max_generation);
32652 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32653 int align_const = get_alignment_constant (gen != large_object_generation);
32659 // Different ways to fail.
32660 BOOL mark_missed_p = FALSE;
32661 BOOL bad_ref_p = FALSE;
32662 BOOL free_ref_p = FALSE;
32668 if (gen != large_object_generation)
32671 gen = large_object_generation;
32672 align_const = get_alignment_constant (gen != large_object_generation);
32673 seg = heap_segment_rw (generation_start_segment (gen));
32682 o = heap_segment_mem (seg);
32683 end = heap_segment_allocated (seg);
32684 //printf ("validating [%Ix-[%Ix\n", o, end);
32689 BOOL marked_p = background_object_marked (o, FALSE);
32693 go_through_object_cl (method_table (o), o, s, oo,
32697 //dprintf (3, ("VOM: verifying member %Ix in obj %Ix", (size_t)*oo, o));
32698 MethodTable *pMT = method_table (*oo);
32700 if (pMT == g_gc_pFreeObjectMethodTable)
32706 if (!pMT->SanityCheck())
32709 dprintf (3, ("Bad member of %Ix %Ix",
32710 (size_t)oo, (size_t)*oo));
32714 if (current_bgc_state == bgc_final_marking)
32716 if (marked_p && !background_object_marked (*oo, FALSE))
32718 mark_missed_p = TRUE;
32727 o = o + Align(s, align_const);
32729 seg = heap_segment_next_rw (seg);
32732 //printf ("didn't find any large object large enough...\n");
32733 //printf ("finished verifying loh\n");
32734 #endif //BACKGROUND_GC
32740 gc_heap::verify_free_lists ()
32742 for (int gen_num = 0; gen_num <= max_generation+1; gen_num++)
32744 dprintf (3, ("Verifying free list for gen:%d", gen_num));
32745 allocator* gen_alloc = generation_allocator (generation_of (gen_num));
32746 size_t sz = gen_alloc->first_bucket_size();
32747 bool verify_undo_slot = (gen_num != 0) && (gen_num != max_generation+1) && !gen_alloc->discard_if_no_fit_p();
32749 for (unsigned int a_l_number = 0; a_l_number < gen_alloc->number_of_buckets(); a_l_number++)
32751 uint8_t* free_list = gen_alloc->alloc_list_head_of (a_l_number);
32755 if (!((CObjectHeader*)free_list)->IsFree())
32757 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't a free object)",
32758 (size_t)free_list));
32761 if (((a_l_number < (gen_alloc->number_of_buckets()-1))&& (unused_array_size (free_list) >= sz))
32762 || ((a_l_number != 0) && (unused_array_size (free_list) < sz/2)))
32764 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't in the right bucket",
32765 (size_t)free_list));
32768 if (verify_undo_slot && (free_list_undo (free_list) != UNDO_EMPTY))
32770 dprintf (3, ("Verifiying Heap: curr free list item %Ix has non empty undo slot",
32771 (size_t)free_list));
32774 if ((gen_num != max_generation+1)&&(object_gennum (free_list)!= gen_num))
32776 dprintf (3, ("Verifiying Heap: curr free list item %Ix is in the wrong generation free list",
32777 (size_t)free_list));
32782 free_list = free_list_slot (free_list);
32784 //verify the sanity of the tail
32785 uint8_t* tail = gen_alloc->alloc_list_tail_of (a_l_number);
32786 if (!((tail == 0) || (tail == prev)))
32788 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
32793 uint8_t* head = gen_alloc->alloc_list_head_of (a_l_number);
32794 if ((head != 0) && (free_list_slot (head) != 0))
32796 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
32807 gc_heap::verify_heap (BOOL begin_gc_p)
32809 int heap_verify_level = static_cast<int>(GCConfig::GetHeapVerifyLevel());
32810 size_t last_valid_brick = 0;
32811 BOOL bCurrentBrickInvalid = FALSE;
32812 BOOL large_brick_p = TRUE;
32813 size_t curr_brick = 0;
32814 size_t prev_brick = (size_t)-1;
32815 int curr_gen_num = max_generation+1;
32816 heap_segment* seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num ) ));
32818 PREFIX_ASSUME(seg != NULL);
32820 uint8_t* curr_object = heap_segment_mem (seg);
32821 uint8_t* prev_object = 0;
32822 uint8_t* begin_youngest = generation_allocation_start(generation_of(0));
32823 uint8_t* end_youngest = heap_segment_allocated (ephemeral_heap_segment);
32824 uint8_t* next_boundary = generation_allocation_start (generation_of (max_generation - 1));
32825 int align_const = get_alignment_constant (FALSE);
32826 size_t total_objects_verified = 0;
32827 size_t total_objects_verified_deep = 0;
32829 #ifdef BACKGROUND_GC
32830 BOOL consider_bgc_mark_p = FALSE;
32831 BOOL check_current_sweep_p = FALSE;
32832 BOOL check_saved_sweep_p = FALSE;
32833 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32834 #endif //BACKGROUND_GC
32836 #ifdef MULTIPLE_HEAPS
32837 t_join* current_join = &gc_t_join;
32838 #ifdef BACKGROUND_GC
32839 if (settings.concurrent && (bgc_thread_id.IsCurrentThread()))
32841 // We always call verify_heap on entry of GC on the SVR GC threads.
32842 current_join = &bgc_t_join;
32844 #endif //BACKGROUND_GC
32845 #endif //MULTIPLE_HEAPS
32847 UNREFERENCED_PARAMETER(begin_gc_p);
32848 #ifdef BACKGROUND_GC
32849 dprintf (2,("[%s]GC#%d(%s): Verifying heap - begin",
32850 (begin_gc_p ? "BEG" : "END"),
32851 VolatileLoad(&settings.gc_index),
32852 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
32854 dprintf (2,("[%s]GC#%d: Verifying heap - begin",
32855 (begin_gc_p ? "BEG" : "END"), VolatileLoad(&settings.gc_index)));
32856 #endif //BACKGROUND_GC
32858 #ifndef MULTIPLE_HEAPS
32859 if ((ephemeral_low != generation_allocation_start (generation_of (max_generation - 1))) ||
32860 (ephemeral_high != heap_segment_reserved (ephemeral_heap_segment)))
32864 #endif //MULTIPLE_HEAPS
32866 #ifdef BACKGROUND_GC
32867 //don't touch the memory because the program is allocating from it.
32868 if (!settings.concurrent)
32869 #endif //BACKGROUND_GC
32871 if (!(heap_verify_level & GCConfig::HEAPVERIFY_NO_MEM_FILL))
32873 //uninit the unused portions of segments.
32874 generation* gen1 = large_object_generation;
32875 heap_segment* seg1 = heap_segment_rw (generation_start_segment (gen1));
32876 PREFIX_ASSUME(seg1 != NULL);
32882 uint8_t* clear_start = heap_segment_allocated (seg1) - plug_skew;
32883 if (heap_segment_used (seg1) > clear_start)
32885 dprintf (3, ("setting end of seg %Ix: [%Ix-[%Ix to 0xaa",
32886 heap_segment_mem (seg1),
32888 heap_segment_used (seg1)));
32889 memset (heap_segment_allocated (seg1) - plug_skew, 0xaa,
32890 (heap_segment_used (seg1) - clear_start));
32892 seg1 = heap_segment_next_rw (seg1);
32896 if (gen1 == large_object_generation)
32898 gen1 = generation_of (max_generation);
32899 seg1 = heap_segment_rw (generation_start_segment (gen1));
32900 PREFIX_ASSUME(seg1 != NULL);
32911 #ifdef MULTIPLE_HEAPS
32912 current_join->join(this, gc_join_verify_copy_table);
32913 if (current_join->joined())
32915 // in concurrent GC, new segment could be allocated when GC is working so the card brick table might not be updated at this point
32916 for (int i = 0; i < n_heaps; i++)
32918 //copy the card and brick tables
32919 if (g_gc_card_table != g_heaps[i]->card_table)
32921 g_heaps[i]->copy_brick_card_table();
32925 current_join->restart();
32928 if (g_gc_card_table != card_table)
32929 copy_brick_card_table();
32930 #endif //MULTIPLE_HEAPS
32932 //verify that the generation structures makes sense
32934 generation* gen = generation_of (max_generation);
32936 assert (generation_allocation_start (gen) ==
32937 heap_segment_mem (heap_segment_rw (generation_start_segment (gen))));
32938 int gen_num = max_generation-1;
32939 generation* prev_gen = gen;
32940 while (gen_num >= 0)
32942 gen = generation_of (gen_num);
32943 assert (generation_allocation_segment (gen) == ephemeral_heap_segment);
32944 assert (generation_allocation_start (gen) >= heap_segment_mem (ephemeral_heap_segment));
32945 assert (generation_allocation_start (gen) < heap_segment_allocated (ephemeral_heap_segment));
32947 if (generation_start_segment (prev_gen ) ==
32948 generation_start_segment (gen))
32950 assert (generation_allocation_start (prev_gen) <
32951 generation_allocation_start (gen));
32960 // Handle segment transitions
32961 if (curr_object >= heap_segment_allocated (seg))
32963 if (curr_object > heap_segment_allocated(seg))
32965 dprintf (3, ("Verifiying Heap: curr_object: %Ix > heap_segment_allocated (seg: %Ix)",
32966 (size_t)curr_object, (size_t)seg));
32969 seg = heap_segment_next_in_range (seg);
32972 #ifdef BACKGROUND_GC
32973 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32974 #endif //BACKGROUND_GC
32975 curr_object = heap_segment_mem(seg);
32981 if (curr_gen_num == (max_generation+1))
32984 seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num)));
32986 PREFIX_ASSUME(seg != NULL);
32988 #ifdef BACKGROUND_GC
32989 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32990 #endif //BACKGROUND_GC
32991 curr_object = heap_segment_mem (seg);
32993 large_brick_p = FALSE;
32994 align_const = get_alignment_constant (TRUE);
32997 break; // Done Verifying Heap -- no more segments
33001 // Are we at the end of the youngest_generation?
33002 if (seg == ephemeral_heap_segment)
33004 if (curr_object >= end_youngest)
33006 // prev_object length is too long if we hit this int3
33007 if (curr_object > end_youngest)
33009 dprintf (3, ("Verifiying Heap: curr_object: %Ix > end_youngest: %Ix",
33010 (size_t)curr_object, (size_t)end_youngest));
33016 if ((curr_object >= next_boundary) && (curr_gen_num > 0))
33019 if (curr_gen_num > 0)
33021 next_boundary = generation_allocation_start (generation_of (curr_gen_num - 1));
33026 //if (is_mark_set (curr_object))
33028 // printf ("curr_object: %Ix is marked!",(size_t)curr_object);
33029 // FATAL_GC_ERROR();
33032 size_t s = size (curr_object);
33033 dprintf (3, ("o: %Ix, s: %d", (size_t)curr_object, s));
33036 dprintf (3, ("Verifying Heap: size of current object %Ix == 0", curr_object));
33040 // If object is not in the youngest generation, then lets
33041 // verify that the brick table is correct....
33042 if (((seg != ephemeral_heap_segment) ||
33043 (brick_of(curr_object) < brick_of(begin_youngest))))
33045 curr_brick = brick_of(curr_object);
33047 // Brick Table Verification...
33049 // On brick transition
33050 // if brick is negative
33051 // verify that brick indirects to previous valid brick
33053 // set current brick invalid flag to be flipped if we
33054 // encounter an object at the correct place
33056 if (curr_brick != prev_brick)
33058 // If the last brick we were examining had positive
33059 // entry but we never found the matching object, then
33060 // we have a problem
33061 // If prev_brick was the last one of the segment
33062 // it's ok for it to be invalid because it is never looked at
33063 if (bCurrentBrickInvalid &&
33064 (curr_brick != brick_of (heap_segment_mem (seg))) &&
33065 !heap_segment_read_only_p (seg))
33067 dprintf (3, ("curr brick %Ix invalid", curr_brick));
33073 //large objects verify the table only if they are in
33075 if ((heap_segment_reserved (seg) <= highest_address) &&
33076 (heap_segment_mem (seg) >= lowest_address) &&
33077 brick_table [curr_brick] != 0)
33079 dprintf (3, ("curr_brick %Ix for large object %Ix not set to -32768",
33080 curr_brick, (size_t)curr_object));
33085 bCurrentBrickInvalid = FALSE;
33090 // If the current brick contains a negative value make sure
33091 // that the indirection terminates at the last valid brick
33092 if (brick_table [curr_brick] <= 0)
33094 if (brick_table [curr_brick] == 0)
33096 dprintf(3, ("curr_brick %Ix for object %Ix set to 0",
33097 curr_brick, (size_t)curr_object));
33100 ptrdiff_t i = curr_brick;
33101 while ((i >= ((ptrdiff_t) brick_of (heap_segment_mem (seg)))) &&
33102 (brick_table[i] < 0))
33104 i = i + brick_table[i];
33106 if (i < ((ptrdiff_t)(brick_of (heap_segment_mem (seg))) - 1))
33108 dprintf (3, ("ptrdiff i: %Ix < brick_of (heap_segment_mem (seg)):%Ix - 1. curr_brick: %Ix",
33109 i, brick_of (heap_segment_mem (seg)),
33113 // if (i != last_valid_brick)
33114 // FATAL_GC_ERROR();
33115 bCurrentBrickInvalid = FALSE;
33117 else if (!heap_segment_read_only_p (seg))
33119 bCurrentBrickInvalid = TRUE;
33124 if (bCurrentBrickInvalid)
33126 if (curr_object == (brick_address(curr_brick) + brick_table[curr_brick] - 1))
33128 bCurrentBrickInvalid = FALSE;
33129 last_valid_brick = curr_brick;
33134 if (*((uint8_t**)curr_object) != (uint8_t *) g_gc_pFreeObjectMethodTable)
33136 #ifdef FEATURE_LOH_COMPACTION
33137 if ((curr_gen_num == (max_generation+1)) && (prev_object != 0))
33139 assert (method_table (prev_object) == g_gc_pFreeObjectMethodTable);
33141 #endif //FEATURE_LOH_COMPACTION
33143 total_objects_verified++;
33145 BOOL can_verify_deep = TRUE;
33146 #ifdef BACKGROUND_GC
33147 can_verify_deep = fgc_should_consider_object (curr_object, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p);
33148 #endif //BACKGROUND_GC
33150 BOOL deep_verify_obj = can_verify_deep;
33151 if ((heap_verify_level & GCConfig::HEAPVERIFY_DEEP_ON_COMPACT) && !settings.compaction)
33152 deep_verify_obj = FALSE;
33154 ((CObjectHeader*)curr_object)->ValidateHeap((Object*)curr_object, deep_verify_obj);
33156 if (can_verify_deep)
33158 if (curr_gen_num > 0)
33160 BOOL need_card_p = FALSE;
33161 if (contain_pointers_or_collectible (curr_object))
33163 dprintf (4, ("curr_object: %Ix", (size_t)curr_object));
33164 size_t crd = card_of (curr_object);
33165 BOOL found_card_p = card_set_p (crd);
33167 #ifdef COLLECTIBLE_CLASS
33168 if (is_collectible(curr_object))
33170 uint8_t* class_obj = get_class_object (curr_object);
33171 if ((class_obj < ephemeral_high) && (class_obj >= next_boundary))
33175 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix pointing to class object %Ix",
33176 card_of (curr_object), (size_t)curr_object, class_obj));
33182 #endif //COLLECTIBLE_CLASS
33184 if (contain_pointers(curr_object))
33186 go_through_object_nostart
33187 (method_table(curr_object), curr_object, s, oo,
33189 if ((crd != card_of ((uint8_t*)oo)) && !found_card_p)
33191 crd = card_of ((uint8_t*)oo);
33192 found_card_p = card_set_p (crd);
33193 need_card_p = FALSE;
33195 if ((*oo < ephemeral_high) && (*oo >= next_boundary))
33197 need_card_p = TRUE;
33200 if (need_card_p && !found_card_p)
33203 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33204 card_of (curr_object), (size_t)curr_object,
33205 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33211 if (need_card_p && !found_card_p)
33213 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33214 card_of (curr_object), (size_t)curr_object,
33215 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33220 total_objects_verified_deep++;
33224 prev_object = curr_object;
33225 prev_brick = curr_brick;
33226 curr_object = curr_object + Align(s, align_const);
33227 if (curr_object < prev_object)
33229 dprintf (3, ("overflow because of a bad object size: %Ix size %Ix", prev_object, s));
33234 #ifdef BACKGROUND_GC
33235 dprintf (2, ("(%s)(%s)(%s) total_objects_verified is %Id, total_objects_verified_deep is %Id",
33236 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p () ? "FGC" : "NGC")),
33237 (begin_gc_p ? "BEG" : "END"),
33238 ((current_c_gc_state == c_gc_state_planning) ? "in plan" : "not in plan"),
33239 total_objects_verified, total_objects_verified_deep));
33240 if (current_c_gc_state != c_gc_state_planning)
33242 assert (total_objects_verified == total_objects_verified_deep);
33244 #endif //BACKGROUND_GC
33246 verify_free_lists();
33248 #ifdef FEATURE_PREMORTEM_FINALIZATION
33249 finalize_queue->CheckFinalizerObjects();
33250 #endif // FEATURE_PREMORTEM_FINALIZATION
33253 // to be consistent with handle table APIs pass a ScanContext*
33254 // to provide the heap number. the SC isn't complete though so
33255 // limit its scope to handle table verification.
33257 sc.thread_number = heap_number;
33258 GCScan::VerifyHandleTable(max_generation, max_generation, &sc);
33261 #ifdef MULTIPLE_HEAPS
33262 current_join->join(this, gc_join_verify_objects_done);
33263 if (current_join->joined())
33264 #endif //MULTIPLE_HEAPS
33266 SyncBlockCache::GetSyncBlockCache()->VerifySyncTableEntry();
33267 #ifdef MULTIPLE_HEAPS
33268 current_join->restart();
33269 #endif //MULTIPLE_HEAPS
33272 #ifdef BACKGROUND_GC
33273 if (!settings.concurrent)
33275 if (current_c_gc_state == c_gc_state_planning)
33277 // temporarily commenting this out 'cause an FGC
33278 // could be triggered before we sweep ephemeral.
33279 //verify_seg_end_mark_array_cleared();
33283 if (settings.concurrent)
33285 verify_mark_array_cleared();
33287 dprintf (2,("GC%d(%s): Verifying heap - end",
33288 VolatileLoad(&settings.gc_index),
33289 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
33291 dprintf (2,("GC#d: Verifying heap - end", VolatileLoad(&settings.gc_index)));
33292 #endif //BACKGROUND_GC
33295 #endif //VERIFY_HEAP
33298 void GCHeap::ValidateObjectMember (Object* obj)
33301 size_t s = size (obj);
33302 uint8_t* o = (uint8_t*)obj;
33304 go_through_object_cl (method_table (obj), o, s, oo,
33306 uint8_t* child_o = *oo;
33309 dprintf (3, ("VOM: m: %Ix obj %Ix", (size_t)child_o, o));
33310 MethodTable *pMT = method_table (child_o);
33312 if (!pMT->SanityCheck()) {
33313 dprintf (3, ("Bad member of %Ix %Ix",
33314 (size_t)oo, (size_t)child_o));
33319 #endif // VERIFY_HEAP
33322 void DestructObject (CObjectHeader* hdr)
33324 UNREFERENCED_PARAMETER(hdr); // compiler bug? -- this *is*, indeed, referenced
33325 hdr->~CObjectHeader();
33328 HRESULT GCHeap::Shutdown ()
33332 GCScan::GcRuntimeStructuresValid (FALSE);
33334 // Cannot assert this, since we use SuspendEE as the mechanism to quiesce all
33335 // threads except the one performing the shutdown.
33336 // ASSERT( !GcInProgress );
33338 // Guard against any more GC occurring and against any threads blocking
33339 // for GC to complete when the GC heap is gone. This fixes a race condition
33340 // where a thread in GC is destroyed as part of process destruction and
33341 // the remaining threads block for GC complete.
33344 //EnterAllocLock();
33346 //EnterFinalizeLock();
33349 // during shutdown lot of threads are suspended
33350 // on this even, we don't want to wake them up just yet
33351 //CloseHandle (WaitForGCEvent);
33353 //find out if the global card table hasn't been used yet
33354 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
33355 if (card_table_refcount (ct) == 0)
33357 destroy_card_table (ct);
33358 g_gc_card_table = nullptr;
33360 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
33361 g_gc_card_bundle_table = nullptr;
33363 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
33364 SoftwareWriteWatch::StaticClose();
33365 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
33368 //destroy all segments on the standby list
33369 while(gc_heap::segment_standby_list != 0)
33371 heap_segment* next_seg = heap_segment_next (gc_heap::segment_standby_list);
33372 #ifdef MULTIPLE_HEAPS
33373 (gc_heap::g_heaps[0])->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
33374 #else //MULTIPLE_HEAPS
33375 pGenGCHeap->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
33376 #endif //MULTIPLE_HEAPS
33377 gc_heap::segment_standby_list = next_seg;
33381 #ifdef MULTIPLE_HEAPS
33383 for (int i = 0; i < gc_heap::n_heaps; i ++)
33385 delete gc_heap::g_heaps[i]->vm_heap;
33386 //destroy pure GC stuff
33387 gc_heap::destroy_gc_heap (gc_heap::g_heaps[i]);
33390 gc_heap::destroy_gc_heap (pGenGCHeap);
33392 #endif //MULTIPLE_HEAPS
33393 gc_heap::shutdown_gc();
33398 // Wait until a garbage collection is complete
33399 // returns NOERROR if wait was OK, other error code if failure.
33400 // WARNING: This will not undo the must complete state. If you are
33401 // in a must complete when you call this, you'd better know what you're
33404 #ifdef FEATURE_PREMORTEM_FINALIZATION
33406 HRESULT AllocateCFinalize(CFinalize **pCFinalize)
33408 *pCFinalize = new (nothrow) CFinalize();
33409 if (*pCFinalize == NULL || !(*pCFinalize)->Initialize())
33410 return E_OUTOFMEMORY;
33414 #endif // FEATURE_PREMORTEM_FINALIZATION
33416 // init the instance heap
33417 HRESULT GCHeap::Init(size_t hn)
33419 HRESULT hres = S_OK;
33421 #ifdef MULTIPLE_HEAPS
33422 if ((pGenGCHeap = gc_heap::make_gc_heap(this, (int)hn)) == 0)
33423 hres = E_OUTOFMEMORY;
33425 UNREFERENCED_PARAMETER(hn);
33426 if (!gc_heap::make_gc_heap())
33427 hres = E_OUTOFMEMORY;
33428 #endif //MULTIPLE_HEAPS
33434 //System wide initialization
33435 HRESULT GCHeap::Initialize ()
33439 g_gc_pFreeObjectMethodTable = GCToEEInterface::GetFreeObjectMethodTable();
33440 g_num_processors = GCToOSInterface::GetTotalProcessorCount();
33441 assert(g_num_processors != 0);
33443 //Initialize the static members.
33446 CreatedObjectCount = 0;
33449 size_t seg_size = get_valid_segment_size();
33450 gc_heap::soh_segment_size = seg_size;
33451 size_t large_seg_size = get_valid_segment_size(TRUE);
33452 gc_heap::min_loh_segment_size = large_seg_size;
33453 gc_heap::min_segment_size = min (seg_size, large_seg_size);
33454 #ifdef SEG_MAPPING_TABLE
33455 gc_heap::min_segment_size_shr = index_of_set_bit (gc_heap::min_segment_size);
33456 #endif //SEG_MAPPING_TABLE
33458 #ifdef MULTIPLE_HEAPS
33459 if (GCConfig::GetNoAffinitize())
33460 gc_heap::gc_thread_no_affinitize_p = true;
33462 uint32_t nhp_from_config = static_cast<uint32_t>(GCConfig::GetHeapCount());
33464 uint32_t nhp_from_process = GCToOSInterface::GetCurrentProcessCpuCount();
33466 uint32_t nhp = ((nhp_from_config == 0) ? nhp_from_process :
33467 (min (nhp_from_config, nhp_from_process)));
33469 nhp = min (nhp, MAX_SUPPORTED_CPUS);
33471 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/, nhp);
33473 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/);
33474 #endif //MULTIPLE_HEAPS
33479 gc_heap::total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit();
33481 gc_heap::mem_one_percent = gc_heap::total_physical_mem / 100;
33482 #ifndef MULTIPLE_HEAPS
33483 gc_heap::mem_one_percent /= g_num_processors;
33484 #endif //!MULTIPLE_HEAPS
33486 // We should only use this if we are in the "many process" mode which really is only applicable
33487 // to very powerful machines - before that's implemented, temporarily I am only enabling this for 80GB+ memory.
33488 // For now I am using an estimate to calculate these numbers but this should really be obtained
33489 // programmatically going forward.
33490 // I am assuming 47 processes using WKS GC and 3 using SVR GC.
33491 // I am assuming 3 in part due to the "very high memory load" is 97%.
33492 int available_mem_th = 10;
33493 if (gc_heap::total_physical_mem >= ((uint64_t)80 * 1024 * 1024 * 1024))
33495 int adjusted_available_mem_th = 3 + (int)((float)47 / (float)(g_num_processors));
33496 available_mem_th = min (available_mem_th, adjusted_available_mem_th);
33499 gc_heap::high_memory_load_th = 100 - available_mem_th;
33502 gc_heap::youngest_gen_desired_th = gc_heap::mem_one_percent;
33505 WaitForGCEvent = new (nothrow) GCEvent;
33507 if (!WaitForGCEvent)
33509 return E_OUTOFMEMORY;
33512 if (!WaitForGCEvent->CreateManualEventNoThrow(TRUE))
33517 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
33518 #if defined (STRESS_HEAP) && !defined (MULTIPLE_HEAPS)
33519 if (GCStress<cfg_any>::IsEnabled()) {
33520 for(int i = 0; i < GCHeap::NUM_HEAP_STRESS_OBJS; i++)
33521 m_StressObjs[i] = CreateGlobalHandle(0);
33522 m_CurStressObj = 0;
33524 #endif //STRESS_HEAP && !MULTIPLE_HEAPS
33525 #endif // FEATURE_REDHAWK
33527 initGCShadow(); // If we are debugging write barriers, initialize heap shadow
33529 #ifdef MULTIPLE_HEAPS
33531 for (unsigned i = 0; i < nhp; i++)
33533 GCHeap* Hp = new (nothrow) GCHeap();
33535 return E_OUTOFMEMORY;
33537 if ((hr = Hp->Init (i))!= S_OK)
33542 // initialize numa node to heap map
33543 heap_select::init_numa_node_to_heap_map(nhp);
33546 #endif //MULTIPLE_HEAPS
33550 GCScan::GcRuntimeStructuresValid (TRUE);
33552 GCToEEInterface::DiagUpdateGenerationBounds();
33559 // GC callback functions
33560 bool GCHeap::IsPromoted(Object* object)
33563 ((CObjectHeader*)object)->Validate();
33566 uint8_t* o = (uint8_t*)object;
33568 if (gc_heap::settings.condemned_generation == max_generation)
33570 #ifdef MULTIPLE_HEAPS
33571 gc_heap* hp = gc_heap::g_heaps[0];
33573 gc_heap* hp = pGenGCHeap;
33574 #endif //MULTIPLE_HEAPS
33576 #ifdef BACKGROUND_GC
33577 if (gc_heap::settings.concurrent)
33579 bool is_marked = (!((o < hp->background_saved_highest_address) && (o >= hp->background_saved_lowest_address))||
33580 hp->background_marked (o));
33584 #endif //BACKGROUND_GC
33586 return (!((o < hp->highest_address) && (o >= hp->lowest_address))
33587 || hp->is_mark_set (o));
33592 gc_heap* hp = gc_heap::heap_of (o);
33593 return (!((o < hp->gc_high) && (o >= hp->gc_low))
33594 || hp->is_mark_set (o));
33598 size_t GCHeap::GetPromotedBytes(int heap_index)
33600 #ifdef BACKGROUND_GC
33601 if (gc_heap::settings.concurrent)
33603 return gc_heap::bpromoted_bytes (heap_index);
33606 #endif //BACKGROUND_GC
33608 return gc_heap::promoted_bytes (heap_index);
33612 unsigned int GCHeap::WhichGeneration (Object* object)
33614 gc_heap* hp = gc_heap::heap_of ((uint8_t*)object);
33615 unsigned int g = hp->object_gennum ((uint8_t*)object);
33616 dprintf (3, ("%Ix is in gen %d", (size_t)object, g));
33620 bool GCHeap::IsEphemeral (Object* object)
33622 uint8_t* o = (uint8_t*)object;
33623 gc_heap* hp = gc_heap::heap_of (o);
33624 return !!hp->ephemeral_pointer_p (o);
33627 // Return NULL if can't find next object. When EE is not suspended,
33628 // the result is not accurate: if the input arg is in gen0, the function could
33629 // return zeroed out memory as next object
33630 Object * GCHeap::NextObj (Object * object)
33633 uint8_t* o = (uint8_t*)object;
33635 #ifndef FEATURE_BASICFREEZE
33636 if (!((o < g_gc_highest_address) && (o >= g_gc_lowest_address)))
33640 #endif //!FEATURE_BASICFREEZE
33642 heap_segment * hs = gc_heap::find_segment (o, FALSE);
33648 BOOL large_object_p = heap_segment_loh_p (hs);
33649 if (large_object_p)
33650 return NULL; //could be racing with another core allocating.
33651 #ifdef MULTIPLE_HEAPS
33652 gc_heap* hp = heap_segment_heap (hs);
33653 #else //MULTIPLE_HEAPS
33655 #endif //MULTIPLE_HEAPS
33656 unsigned int g = hp->object_gennum ((uint8_t*)object);
33657 if ((g == 0) && hp->settings.demotion)
33658 return NULL;//could be racing with another core allocating.
33659 int align_const = get_alignment_constant (!large_object_p);
33660 uint8_t* nextobj = o + Align (size (o), align_const);
33661 if (nextobj <= o) // either overflow or 0 sized object.
33666 if ((nextobj < heap_segment_mem(hs)) ||
33667 (nextobj >= heap_segment_allocated(hs) && hs != hp->ephemeral_heap_segment) ||
33668 (nextobj >= hp->alloc_allocated))
33673 return (Object *)nextobj;
33676 #endif // VERIFY_HEAP
33681 #ifdef FEATURE_BASICFREEZE
33682 BOOL GCHeap::IsInFrozenSegment (Object * object)
33684 uint8_t* o = (uint8_t*)object;
33685 heap_segment * hs = gc_heap::find_segment (o, FALSE);
33686 //We create a frozen object for each frozen segment before the segment is inserted
33687 //to segment list; during ngen, we could also create frozen objects in segments which
33688 //don't belong to current GC heap.
33689 //So we return true if hs is NULL. It might create a hole about detecting invalidate
33690 //object. But given all other checks present, the hole should be very small
33691 return !hs || heap_segment_read_only_p (hs);
33693 #endif //FEATURE_BASICFREEZE
33695 #endif //VERIFY_HEAP
33697 // returns TRUE if the pointer is in one of the GC heaps.
33698 bool GCHeap::IsHeapPointer (void* vpObject, bool small_heap_only)
33700 STATIC_CONTRACT_SO_TOLERANT;
33702 // removed STATIC_CONTRACT_CAN_TAKE_LOCK here because find_segment
33703 // no longer calls GCEvent::Wait which eventually takes a lock.
33705 uint8_t* object = (uint8_t*) vpObject;
33706 #ifndef FEATURE_BASICFREEZE
33707 if (!((object < g_gc_highest_address) && (object >= g_gc_lowest_address)))
33709 #endif //!FEATURE_BASICFREEZE
33711 heap_segment * hs = gc_heap::find_segment (object, small_heap_only);
33715 #ifdef STRESS_PINNING
33716 static n_promote = 0;
33717 #endif //STRESS_PINNING
33718 // promote an object
33719 void GCHeap::Promote(Object** ppObject, ScanContext* sc, uint32_t flags)
33721 THREAD_NUMBER_FROM_CONTEXT;
33722 #ifndef MULTIPLE_HEAPS
33723 const int thread = 0;
33724 #endif //!MULTIPLE_HEAPS
33726 uint8_t* o = (uint8_t*)*ppObject;
33731 #ifdef DEBUG_DestroyedHandleValue
33732 // we can race with destroy handle during concurrent scan
33733 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
33735 #endif //DEBUG_DestroyedHandleValue
33739 gc_heap* hp = gc_heap::heap_of (o);
33741 dprintf (3, ("Promote %Ix", (size_t)o));
33743 #ifdef INTERIOR_POINTERS
33744 if (flags & GC_CALL_INTERIOR)
33746 if ((o < hp->gc_low) || (o >= hp->gc_high))
33750 if ( (o = hp->find_object (o, hp->gc_low)) == 0)
33756 #endif //INTERIOR_POINTERS
33758 #ifdef FEATURE_CONSERVATIVE_GC
33759 // For conservative GC, a value on stack may point to middle of a free object.
33760 // In this case, we don't need to promote the pointer.
33761 if (GCConfig::GetConservativeGC()
33762 && ((CObjectHeader*)o)->IsFree())
33769 ((CObjectHeader*)o)->ValidatePromote(sc, flags);
33771 UNREFERENCED_PARAMETER(sc);
33774 if (flags & GC_CALL_PINNED)
33775 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
33777 #ifdef STRESS_PINNING
33778 if ((++n_promote % 20) == 1)
33779 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
33780 #endif //STRESS_PINNING
33782 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
33783 size_t promoted_size_begin = hp->promoted_bytes (thread);
33784 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
33786 if ((o >= hp->gc_low) && (o < hp->gc_high))
33788 hpt->mark_object_simple (&o THREAD_NUMBER_ARG);
33791 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
33792 size_t promoted_size_end = hp->promoted_bytes (thread);
33795 if (sc->pCurrentDomain)
33797 sc->pCurrentDomain->RecordSurvivedBytes ((promoted_size_end - promoted_size_begin), thread);
33800 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
33802 STRESS_LOG_ROOT_PROMOTE(ppObject, o, o ? header(o)->GetMethodTable() : NULL);
33805 void GCHeap::Relocate (Object** ppObject, ScanContext* sc,
33808 UNREFERENCED_PARAMETER(sc);
33810 uint8_t* object = (uint8_t*)(Object*)(*ppObject);
33812 THREAD_NUMBER_FROM_CONTEXT;
33814 //dprintf (3, ("Relocate location %Ix\n", (size_t)ppObject));
33815 dprintf (3, ("R: %Ix", (size_t)ppObject));
33820 gc_heap* hp = gc_heap::heap_of (object);
33823 if (!(flags & GC_CALL_INTERIOR))
33825 // We cannot validate this object if it's in the condemned gen because it could
33826 // be one of the objects that were overwritten by an artificial gap due to a pinned plug.
33827 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
33829 ((CObjectHeader*)object)->Validate(FALSE);
33834 dprintf (3, ("Relocate %Ix\n", (size_t)object));
33838 if ((flags & GC_CALL_INTERIOR) && gc_heap::settings.loh_compaction)
33840 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
33845 if (gc_heap::loh_object_p (object))
33847 pheader = hp->find_object (object, 0);
33853 ptrdiff_t ref_offset = object - pheader;
33854 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
33855 *ppObject = (Object*)(pheader + ref_offset);
33862 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
33863 *ppObject = (Object*)pheader;
33866 STRESS_LOG_ROOT_RELOCATE(ppObject, object, pheader, ((!(flags & GC_CALL_INTERIOR)) ? ((Object*)object)->GetGCSafeMethodTable() : 0));
33869 /*static*/ bool GCHeap::IsObjectInFixedHeap(Object *pObj)
33871 // For now we simply look at the size of the object to determine if it in the
33872 // fixed heap or not. If the bit indicating this gets set at some point
33873 // we should key off that instead.
33874 return size( pObj ) >= LARGE_OBJECT_SIZE;
33877 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
33880 void StressHeapDummy ();
33882 static int32_t GCStressStartCount = -1;
33883 static int32_t GCStressCurCount = 0;
33884 static int32_t GCStressStartAtJit = -1;
33886 // the maximum number of foreground GCs we'll induce during one BGC
33887 // (this number does not include "naturally" occuring GCs).
33888 static int32_t GCStressMaxFGCsPerBGC = -1;
33890 // CLRRandom implementation can produce FPU exceptions if
33891 // the test/application run by CLR is enabling any FPU exceptions.
33892 // We want to avoid any unexpected exception coming from stress
33893 // infrastructure, so CLRRandom is not an option.
33894 // The code below is a replicate of CRT rand() implementation.
33895 // Using CRT rand() is not an option because we will interfere with the user application
33896 // that may also use it.
33897 int StressRNG(int iMaxValue)
33899 static BOOL bisRandInit = FALSE;
33900 static int lHoldrand = 1L;
33904 lHoldrand = (int)time(NULL);
33905 bisRandInit = TRUE;
33907 int randValue = (((lHoldrand = lHoldrand * 214013L + 2531011L) >> 16) & 0x7fff);
33908 return randValue % iMaxValue;
33910 #endif // STRESS_HEAP
33911 #endif // !FEATURE_REDHAWK
33913 // free up object so that things will move and then do a GC
33914 //return TRUE if GC actually happens, otherwise FALSE
33915 bool GCHeap::StressHeap(gc_alloc_context * context)
33917 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
33918 alloc_context* acontext = static_cast<alloc_context*>(context);
33919 assert(context != nullptr);
33921 // if GC stress was dynamically disabled during this run we return FALSE
33922 if (!GCStressPolicy::IsEnabled())
33926 if (g_pConfig->FastGCStressLevel() && !GCToEEInterface::GetThread()->StressHeapIsEnabled()) {
33932 if ((g_pConfig->GetGCStressLevel() & EEConfig::GCSTRESS_UNIQUE)
33934 || g_pConfig->FastGCStressLevel() > 1
33937 if (!Thread::UniqueStack(&acontext)) {
33942 #ifdef BACKGROUND_GC
33943 // don't trigger a GC from the GC threads but still trigger GCs from user threads.
33944 if (GCToEEInterface::WasCurrentThreadCreatedByGC())
33948 #endif //BACKGROUND_GC
33950 if (GCStressStartAtJit == -1 || GCStressStartCount == -1)
33952 GCStressStartCount = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_GCStressStart);
33953 GCStressStartAtJit = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressStartAtJit);
33956 if (GCStressMaxFGCsPerBGC == -1)
33958 GCStressMaxFGCsPerBGC = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressMaxFGCsPerBGC);
33959 if (g_pConfig->IsGCStressMix() && GCStressMaxFGCsPerBGC == -1)
33960 GCStressMaxFGCsPerBGC = 6;
33964 if (g_JitCount < GCStressStartAtJit)
33968 // Allow programmer to skip the first N Stress GCs so that you can
33969 // get to the interesting ones faster.
33970 Interlocked::Increment(&GCStressCurCount);
33971 if (GCStressCurCount < GCStressStartCount)
33974 // throttle the number of stress-induced GCs by a factor given by GCStressStep
33975 if ((GCStressCurCount % g_pConfig->GetGCStressStep()) != 0)
33980 #ifdef BACKGROUND_GC
33981 if (IsConcurrentGCEnabled() && IsConcurrentGCInProgress())
33983 // allow a maximum number of stress induced FGCs during one BGC
33984 if (gc_stress_fgcs_in_bgc >= GCStressMaxFGCsPerBGC)
33986 ++gc_stress_fgcs_in_bgc;
33988 #endif // BACKGROUND_GC
33990 if (g_pStringClass == 0)
33992 // If the String class has not been loaded, dont do any stressing. This should
33993 // be kept to a minimum to get as complete coverage as possible.
33994 _ASSERTE(g_fEEInit);
33998 #ifndef MULTIPLE_HEAPS
33999 static int32_t OneAtATime = -1;
34001 // Only bother with this if the stress level is big enough and if nobody else is
34002 // doing it right now. Note that some callers are inside the AllocLock and are
34003 // guaranteed synchronized. But others are using AllocationContexts and have no
34004 // particular synchronization.
34006 // For this latter case, we want a very high-speed way of limiting this to one
34007 // at a time. A secondary advantage is that we release part of our StressObjs
34008 // buffer sparingly but just as effectively.
34010 if (Interlocked::Increment(&OneAtATime) == 0 &&
34011 !TrackAllocations()) // Messing with object sizes can confuse the profiler (see ICorProfilerInfo::GetObjectSize)
34015 // If the current string is used up
34016 if (HndFetchHandle(m_StressObjs[m_CurStressObj]) == 0)
34018 // Populate handles with strings
34019 int i = m_CurStressObj;
34020 while(HndFetchHandle(m_StressObjs[i]) == 0)
34022 _ASSERTE(m_StressObjs[i] != 0);
34023 unsigned strLen = (LARGE_OBJECT_SIZE - 32) / sizeof(WCHAR);
34024 unsigned strSize = PtrAlign(StringObject::GetSize(strLen));
34026 // update the cached type handle before allocating
34027 SetTypeHandleOnThreadForAlloc(TypeHandle(g_pStringClass));
34028 str = (StringObject*) pGenGCHeap->allocate (strSize, acontext);
34031 str->SetMethodTable (g_pStringClass);
34032 str->SetStringLength (strLen);
34034 HndAssignHandle(m_StressObjs[i], ObjectToOBJECTREF(str));
34036 i = (i + 1) % NUM_HEAP_STRESS_OBJS;
34037 if (i == m_CurStressObj) break;
34040 // advance the current handle to the next string
34041 m_CurStressObj = (m_CurStressObj + 1) % NUM_HEAP_STRESS_OBJS;
34044 // Get the current string
34045 str = (StringObject*) OBJECTREFToObject(HndFetchHandle(m_StressObjs[m_CurStressObj]));
34048 // Chop off the end of the string and form a new object out of it.
34049 // This will 'free' an object at the begining of the heap, which will
34050 // force data movement. Note that we can only do this so many times.
34051 // before we have to move on to the next string.
34052 unsigned sizeOfNewObj = (unsigned)Align(min_obj_size * 31);
34053 if (str->GetStringLength() > sizeOfNewObj / sizeof(WCHAR))
34055 unsigned sizeToNextObj = (unsigned)Align(size(str));
34056 uint8_t* freeObj = ((uint8_t*) str) + sizeToNextObj - sizeOfNewObj;
34057 pGenGCHeap->make_unused_array (freeObj, sizeOfNewObj);
34058 str->SetStringLength(str->GetStringLength() - (sizeOfNewObj / sizeof(WCHAR)));
34062 // Let the string itself become garbage.
34063 // will be realloced next time around
34064 HndAssignHandle(m_StressObjs[m_CurStressObj], 0);
34068 Interlocked::Decrement(&OneAtATime);
34069 #endif // !MULTIPLE_HEAPS
34070 if (IsConcurrentGCEnabled())
34072 int rgen = StressRNG(10);
34074 // gen0:gen1:gen2 distribution: 40:40:20
34077 else if (rgen >= 4)
34082 GarbageCollectTry (rgen, FALSE, collection_gcstress);
34086 GarbageCollect(max_generation, FALSE, collection_gcstress);
34091 UNREFERENCED_PARAMETER(context);
34093 #endif // defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34097 #ifdef FEATURE_PREMORTEM_FINALIZATION
34098 #define REGISTER_FOR_FINALIZATION(_object, _size) \
34099 hp->finalize_queue->RegisterForFinalization (0, (_object), (_size))
34100 #else // FEATURE_PREMORTEM_FINALIZATION
34101 #define REGISTER_FOR_FINALIZATION(_object, _size) true
34102 #endif // FEATURE_PREMORTEM_FINALIZATION
34104 #define CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(_object, _size, _register) do { \
34105 if ((_object) == NULL || ((_register) && !REGISTER_FOR_FINALIZATION(_object, _size))) \
34107 STRESS_LOG_OOM_STACK(_size); \
34113 // Small Object Allocator
34116 // Allocate small object with an alignment requirement of 8-bytes.
34118 GCHeap::AllocAlign8(gc_alloc_context* ctx, size_t size, uint32_t flags )
34120 #ifdef FEATURE_64BIT_ALIGNMENT
34126 alloc_context* acontext = static_cast<alloc_context*>(ctx);
34128 #ifdef MULTIPLE_HEAPS
34129 if (acontext->get_alloc_heap() == 0)
34131 AssignHeap (acontext);
34132 assert (acontext->get_alloc_heap());
34135 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
34137 gc_heap* hp = pGenGCHeap;
34138 #endif //MULTIPLE_HEAPS
34140 return AllocAlign8Common(hp, acontext, size, flags);
34142 UNREFERENCED_PARAMETER(ctx);
34143 UNREFERENCED_PARAMETER(size);
34144 UNREFERENCED_PARAMETER(flags);
34145 assert(!"should not call GCHeap::AllocAlign8 without FEATURE_64BIT_ALIGNMENT defined!");
34147 #endif //FEATURE_64BIT_ALIGNMENT
34150 // Common code used by both variants of AllocAlign8 above.
34152 GCHeap::AllocAlign8Common(void* _hp, alloc_context* acontext, size_t size, uint32_t flags)
34154 #ifdef FEATURE_64BIT_ALIGNMENT
34160 gc_heap* hp = (gc_heap*)_hp;
34164 Object* newAlloc = NULL;
34167 #ifdef COUNT_CYCLES
34168 AllocStart = GetCycleCount32();
34170 #elif defined(ENABLE_INSTRUMENTATION)
34171 unsigned AllocStart = GetInstLogTime();
34173 #endif //COUNT_CYCLES
34176 if (size < LARGE_OBJECT_SIZE)
34182 // Depending on where in the object the payload requiring 8-byte alignment resides we might have to
34183 // align the object header on an 8-byte boundary or midway between two such boundaries. The unaligned
34184 // case is indicated to the GC via the GC_ALLOC_ALIGN8_BIAS flag.
34185 size_t desiredAlignment = (flags & GC_ALLOC_ALIGN8_BIAS) ? 4 : 0;
34187 // Retrieve the address of the next allocation from the context (note that we're inside the alloc
34188 // lock at this point).
34189 uint8_t* result = acontext->alloc_ptr;
34191 // Will an allocation at this point yield the correct alignment and fit into the remainder of the
34193 if ((((size_t)result & 7) == desiredAlignment) && ((result + size) <= acontext->alloc_limit))
34195 // Yes, we can just go ahead and make the allocation.
34196 newAlloc = (Object*) hp->allocate (size, acontext);
34197 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
34201 // No, either the next available address is not aligned in the way we require it or there's
34202 // not enough space to allocate an object of the required size. In both cases we allocate a
34203 // padding object (marked as a free object). This object's size is such that it will reverse
34204 // the alignment of the next header (asserted below).
34206 // We allocate both together then decide based on the result whether we'll format the space as
34207 // free object + real object or real object + free object.
34208 ASSERT((Align(min_obj_size) & 7) == 4);
34209 CObjectHeader *freeobj = (CObjectHeader*) hp->allocate (Align(size) + Align(min_obj_size), acontext);
34212 if (((size_t)freeobj & 7) == desiredAlignment)
34214 // New allocation has desired alignment, return this one and place the free object at the
34215 // end of the allocated space.
34216 newAlloc = (Object*)freeobj;
34217 freeobj = (CObjectHeader*)((uint8_t*)freeobj + Align(size));
34221 // New allocation is still mis-aligned, format the initial space as a free object and the
34222 // rest of the space should be correctly aligned for the real object.
34223 newAlloc = (Object*)((uint8_t*)freeobj + Align(min_obj_size));
34224 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
34226 freeobj->SetFree(min_obj_size);
34232 // The LOH always guarantees at least 8-byte alignment, regardless of platform. Moreover it doesn't
34233 // support mis-aligned object headers so we can't support biased headers as above. Luckily for us
34234 // we've managed to arrange things so the only case where we see a bias is for boxed value types and
34235 // these can never get large enough to be allocated on the LOH.
34236 ASSERT(65536 < LARGE_OBJECT_SIZE);
34237 ASSERT((flags & GC_ALLOC_ALIGN8_BIAS) == 0);
34239 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
34241 newAlloc = (Object*) hp->allocate_large_object (size, acontext->alloc_bytes_loh);
34242 ASSERT(((size_t)newAlloc & 7) == 0);
34245 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
34248 #ifdef COUNT_CYCLES
34249 finish = GetCycleCount32();
34250 #elif defined(ENABLE_INSTRUMENTATION)
34251 finish = GetInstLogTime();
34252 #endif //COUNT_CYCLES
34253 AllocDuration += finish - AllocStart;
34258 UNREFERENCED_PARAMETER(_hp);
34259 UNREFERENCED_PARAMETER(acontext);
34260 UNREFERENCED_PARAMETER(size);
34261 UNREFERENCED_PARAMETER(flags);
34262 assert(!"Should not call GCHeap::AllocAlign8Common without FEATURE_64BIT_ALIGNMENT defined!");
34264 #endif // FEATURE_64BIT_ALIGNMENT
34268 GCHeap::AllocLHeap( size_t size, uint32_t flags REQD_ALIGN_DCL)
34277 Object* newAlloc = NULL;
34280 #ifdef COUNT_CYCLES
34281 AllocStart = GetCycleCount32();
34283 #elif defined(ENABLE_INSTRUMENTATION)
34284 unsigned AllocStart = GetInstLogTime();
34286 #endif //COUNT_CYCLES
34289 #ifdef MULTIPLE_HEAPS
34290 //take the first heap....
34291 gc_heap* hp = gc_heap::g_heaps[0];
34293 gc_heap* hp = pGenGCHeap;
34295 // prefix complains about us dereferencing hp in wks build even though we only access static members
34296 // this way. not sure how to shut it up except for this ugly workaround:
34297 PREFIX_ASSUME(hp != NULL);
34299 #endif //MULTIPLE_HEAPS
34301 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
34303 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
34304 #ifdef FEATURE_STRUCTALIGN
34305 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
34306 #endif // FEATURE_STRUCTALIGN
34307 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
34310 #ifdef COUNT_CYCLES
34311 finish = GetCycleCount32();
34312 #elif defined(ENABLE_INSTRUMENTATION)
34313 finish = GetInstLogTime();
34314 #endif //COUNT_CYCLES
34315 AllocDuration += finish - AllocStart;
34322 GCHeap::Alloc(gc_alloc_context* context, size_t size, uint32_t flags REQD_ALIGN_DCL)
34331 Object* newAlloc = NULL;
34332 alloc_context* acontext = static_cast<alloc_context*>(context);
34335 #ifdef COUNT_CYCLES
34336 AllocStart = GetCycleCount32();
34338 #elif defined(ENABLE_INSTRUMENTATION)
34339 unsigned AllocStart = GetInstLogTime();
34341 #endif //COUNT_CYCLES
34344 #ifdef MULTIPLE_HEAPS
34345 if (acontext->get_alloc_heap() == 0)
34347 AssignHeap (acontext);
34348 assert (acontext->get_alloc_heap());
34350 #endif //MULTIPLE_HEAPS
34352 #ifdef MULTIPLE_HEAPS
34353 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
34355 gc_heap* hp = pGenGCHeap;
34357 // prefix complains about us dereferencing hp in wks build even though we only access static members
34358 // this way. not sure how to shut it up except for this ugly workaround:
34359 PREFIX_ASSUME(hp != NULL);
34361 #endif //MULTIPLE_HEAPS
34363 if (size < LARGE_OBJECT_SIZE)
34369 newAlloc = (Object*) hp->allocate (size + ComputeMaxStructAlignPad(requiredAlignment), acontext);
34370 #ifdef FEATURE_STRUCTALIGN
34371 newAlloc = (Object*) hp->pad_for_alignment ((uint8_t*) newAlloc, requiredAlignment, size, acontext);
34372 #endif // FEATURE_STRUCTALIGN
34373 // ASSERT (newAlloc);
34377 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
34378 #ifdef FEATURE_STRUCTALIGN
34379 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
34380 #endif // FEATURE_STRUCTALIGN
34383 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
34386 #ifdef COUNT_CYCLES
34387 finish = GetCycleCount32();
34388 #elif defined(ENABLE_INSTRUMENTATION)
34389 finish = GetInstLogTime();
34390 #endif //COUNT_CYCLES
34391 AllocDuration += finish - AllocStart;
34398 GCHeap::FixAllocContext (gc_alloc_context* context, bool lockp, void* arg, void *heap)
34400 alloc_context* acontext = static_cast<alloc_context*>(context);
34401 #ifdef MULTIPLE_HEAPS
34404 acontext->alloc_count = 0;
34406 uint8_t * alloc_ptr = acontext->alloc_ptr;
34411 // The acontext->alloc_heap can be out of sync with the ptrs because
34412 // of heap re-assignment in allocate
34413 gc_heap* hp = gc_heap::heap_of (alloc_ptr);
34415 gc_heap* hp = pGenGCHeap;
34416 #endif //MULTIPLE_HEAPS
34418 if (heap == NULL || heap == hp)
34422 enter_spin_lock (&hp->more_space_lock);
34424 hp->fix_allocation_context (acontext, ((arg != 0)? TRUE : FALSE),
34425 get_alignment_constant(TRUE));
34428 leave_spin_lock (&hp->more_space_lock);
34434 GCHeap::GetContainingObject (void *pInteriorPtr, bool fCollectedGenOnly)
34436 uint8_t *o = (uint8_t*)pInteriorPtr;
34438 gc_heap* hp = gc_heap::heap_of (o);
34440 uint8_t* lowest = (fCollectedGenOnly ? hp->gc_low : hp->lowest_address);
34441 uint8_t* highest = (fCollectedGenOnly ? hp->gc_high : hp->highest_address);
34443 if (o >= lowest && o < highest)
34445 o = hp->find_object (o, lowest);
34452 return (Object *)o;
34455 BOOL should_collect_optimized (dynamic_data* dd, BOOL low_memory_p)
34457 if (dd_new_allocation (dd) < 0)
34462 if (((float)(dd_new_allocation (dd)) / (float)dd_desired_allocation (dd)) < (low_memory_p ? 0.7 : 0.3))
34470 //----------------------------------------------------------------------------
34471 // #GarbageCollector
34473 // API to ensure that a complete new garbage collection takes place
34476 GCHeap::GarbageCollect (int generation, bool low_memory_p, int mode)
34481 size_t total_allocated = 0;
34482 size_t total_desired = 0;
34483 #ifdef MULTIPLE_HEAPS
34485 for (hn = 0; hn < gc_heap::n_heaps; hn++)
34487 gc_heap* hp = gc_heap::g_heaps [hn];
34488 total_desired += dd_desired_allocation (hp->dynamic_data_of (0));
34489 total_allocated += dd_desired_allocation (hp->dynamic_data_of (0))-
34490 dd_new_allocation (hp->dynamic_data_of (0));
34493 gc_heap* hp = pGenGCHeap;
34494 total_desired = dd_desired_allocation (hp->dynamic_data_of (0));
34495 total_allocated = dd_desired_allocation (hp->dynamic_data_of (0))-
34496 dd_new_allocation (hp->dynamic_data_of (0));
34497 #endif //MULTIPLE_HEAPS
34499 if ((total_desired > gc_heap::mem_one_percent) && (total_allocated < gc_heap::mem_one_percent))
34501 dprintf (2, ("Async low mem but we've only allocated %d (< 10%% of physical mem) out of %d, returning",
34502 total_allocated, total_desired));
34509 #ifdef MULTIPLE_HEAPS
34510 gc_heap* hpt = gc_heap::g_heaps[0];
34513 #endif //MULTIPLE_HEAPS
34515 generation = (generation < 0) ? max_generation : min (generation, max_generation);
34516 dynamic_data* dd = hpt->dynamic_data_of (generation);
34518 #ifdef BACKGROUND_GC
34519 if (recursive_gc_sync::background_running_p())
34521 if ((mode == collection_optimized) || (mode & collection_non_blocking))
34525 if (mode & collection_blocking)
34527 pGenGCHeap->background_gc_wait();
34528 if (mode & collection_optimized)
34534 #endif //BACKGROUND_GC
34536 if (mode & collection_optimized)
34538 if (pGenGCHeap->gc_started)
34544 BOOL should_collect = FALSE;
34545 BOOL should_check_loh = (generation == max_generation);
34546 #ifdef MULTIPLE_HEAPS
34547 for (int i = 0; i < gc_heap::n_heaps; i++)
34549 dynamic_data* dd1 = gc_heap::g_heaps [i]->dynamic_data_of (generation);
34550 dynamic_data* dd2 = (should_check_loh ?
34551 (gc_heap::g_heaps [i]->dynamic_data_of (max_generation + 1)) :
34554 if (should_collect_optimized (dd1, low_memory_p))
34556 should_collect = TRUE;
34559 if (dd2 && should_collect_optimized (dd2, low_memory_p))
34561 should_collect = TRUE;
34566 should_collect = should_collect_optimized (dd, low_memory_p);
34567 if (!should_collect && should_check_loh)
34570 should_collect_optimized (hpt->dynamic_data_of (max_generation + 1), low_memory_p);
34572 #endif //MULTIPLE_HEAPS
34573 if (!should_collect)
34580 size_t CollectionCountAtEntry = dd_collection_count (dd);
34581 size_t BlockingCollectionCountAtEntry = gc_heap::full_gc_counts[gc_type_blocking];
34582 size_t CurrentCollectionCount = 0;
34586 CurrentCollectionCount = GarbageCollectTry(generation, low_memory_p, mode);
34588 if ((mode & collection_blocking) &&
34589 (generation == max_generation) &&
34590 (gc_heap::full_gc_counts[gc_type_blocking] == BlockingCollectionCountAtEntry))
34592 #ifdef BACKGROUND_GC
34593 if (recursive_gc_sync::background_running_p())
34595 pGenGCHeap->background_gc_wait();
34597 #endif //BACKGROUND_GC
34602 if (CollectionCountAtEntry == CurrentCollectionCount)
34611 GCHeap::GarbageCollectTry (int generation, BOOL low_memory_p, int mode)
34613 int gen = (generation < 0) ?
34614 max_generation : min (generation, max_generation);
34616 gc_reason reason = reason_empty;
34620 if (mode & collection_blocking)
34621 reason = reason_lowmemory_blocking;
34623 reason = reason_lowmemory;
34626 reason = reason_induced;
34628 if (reason == reason_induced)
34630 if (mode & collection_compacting)
34632 reason = reason_induced_compacting;
34634 else if (mode & collection_non_blocking)
34636 reason = reason_induced_noforce;
34639 else if (mode & collection_gcstress)
34641 reason = reason_gcstress;
34646 return GarbageCollectGeneration (gen, reason);
34649 void gc_heap::do_pre_gc()
34651 STRESS_LOG_GC_STACK;
34654 STRESS_LOG_GC_START(VolatileLoad(&settings.gc_index),
34655 (uint32_t)settings.condemned_generation,
34656 (uint32_t)settings.reason);
34657 #endif // STRESS_LOG
34659 #ifdef MULTIPLE_HEAPS
34660 gc_heap* hp = g_heaps[0];
34663 #endif //MULTIPLE_HEAPS
34665 #ifdef BACKGROUND_GC
34666 settings.b_state = hp->current_bgc_state;
34667 #endif //BACKGROUND_GC
34669 #ifdef BACKGROUND_GC
34670 dprintf (1, ("*GC* %d(gen0:%d)(%d)(%s)(%d)",
34671 VolatileLoad(&settings.gc_index),
34672 dd_collection_count (hp->dynamic_data_of (0)),
34673 settings.condemned_generation,
34674 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")),
34675 settings.b_state));
34677 dprintf (1, ("*GC* %d(gen0:%d)(%d)",
34678 VolatileLoad(&settings.gc_index),
34679 dd_collection_count(hp->dynamic_data_of(0)),
34680 settings.condemned_generation));
34681 #endif //BACKGROUND_GC
34683 // TODO: this can happen...it's because of the way we are calling
34684 // do_pre_gc, will fix later.
34685 //if (last_gc_index > VolatileLoad(&settings.gc_index))
34687 // FATAL_GC_ERROR();
34690 last_gc_index = VolatileLoad(&settings.gc_index);
34691 GCHeap::UpdatePreGCCounters();
34693 if (settings.concurrent)
34695 #ifdef BACKGROUND_GC
34696 full_gc_counts[gc_type_background]++;
34697 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34698 GCHeap::gc_stress_fgcs_in_bgc = 0;
34699 #endif // STRESS_HEAP && !FEATURE_REDHAWK
34700 #endif // BACKGROUND_GC
34704 if (settings.condemned_generation == max_generation)
34706 full_gc_counts[gc_type_blocking]++;
34710 #ifdef BACKGROUND_GC
34711 if (settings.background_p)
34713 ephemeral_fgc_counts[settings.condemned_generation]++;
34715 #endif //BACKGROUND_GC
34719 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34722 SystemDomain::ResetADSurvivedBytes();
34724 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34727 #ifdef GC_CONFIG_DRIVEN
34728 void gc_heap::record_interesting_info_per_heap()
34730 // datapoints are always from the last blocking GC so don't record again
34732 if (!(settings.concurrent))
34734 for (int i = 0; i < max_idp_count; i++)
34736 interesting_data_per_heap[i] += interesting_data_per_gc[i];
34740 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
34741 if (compact_reason >= 0)
34742 (compact_reasons_per_heap[compact_reason])++;
34743 int expand_mechanism = get_gc_data_per_heap()->get_mechanism (gc_heap_expand);
34744 if (expand_mechanism >= 0)
34745 (expand_mechanisms_per_heap[expand_mechanism])++;
34747 for (int i = 0; i < max_gc_mechanism_bits_count; i++)
34749 if (get_gc_data_per_heap()->is_mechanism_bit_set ((gc_mechanism_bit_per_heap)i))
34750 (interesting_mechanism_bits_per_heap[i])++;
34753 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
34754 cprintf (("%2d | %6d | %1d | %1s | %2s | %2s | %2s | %2s | %2s || %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id |",
34756 (size_t)settings.gc_index,
34757 settings.condemned_generation,
34758 // TEMP - I am just doing this for wks GC 'cuase I wanna see the pattern of doing C/S GCs.
34759 (settings.compaction ? (((compact_reason >= 0) && gc_heap_compact_reason_mandatory_p[compact_reason]) ? "M" : "W") : ""), // compaction
34760 ((expand_mechanism >= 0)? "X" : ""), // EX
34761 ((expand_mechanism == expand_reuse_normal) ? "X" : ""), // NF
34762 ((expand_mechanism == expand_reuse_bestfit) ? "X" : ""), // BF
34763 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_mark_list_bit) ? "X" : ""), // ML
34764 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_demotion_bit) ? "X" : ""), // DM
34765 interesting_data_per_gc[idp_pre_short],
34766 interesting_data_per_gc[idp_post_short],
34767 interesting_data_per_gc[idp_merged_pin],
34768 interesting_data_per_gc[idp_converted_pin],
34769 interesting_data_per_gc[idp_pre_pin],
34770 interesting_data_per_gc[idp_post_pin],
34771 interesting_data_per_gc[idp_pre_and_post_pin],
34772 interesting_data_per_gc[idp_pre_short_padded],
34773 interesting_data_per_gc[idp_post_short_padded]));
34776 void gc_heap::record_global_mechanisms()
34778 for (int i = 0; i < max_global_mechanisms_count; i++)
34780 if (gc_data_global.get_mechanism_p ((gc_global_mechanism_p)i))
34782 ::record_global_mechanism (i);
34787 BOOL gc_heap::should_do_sweeping_gc (BOOL compact_p)
34789 if (!compact_ratio)
34790 return (!compact_p);
34792 size_t compact_count = compact_or_sweep_gcs[0];
34793 size_t sweep_count = compact_or_sweep_gcs[1];
34795 size_t total_count = compact_count + sweep_count;
34796 BOOL should_compact = compact_p;
34797 if (total_count > 3)
34801 int temp_ratio = (int)((compact_count + 1) * 100 / (total_count + 1));
34802 if (temp_ratio > compact_ratio)
34804 // cprintf (("compact would be: %d, total_count: %d, ratio would be %d%% > target\n",
34805 // (compact_count + 1), (total_count + 1), temp_ratio));
34806 should_compact = FALSE;
34811 int temp_ratio = (int)((sweep_count + 1) * 100 / (total_count + 1));
34812 if (temp_ratio > (100 - compact_ratio))
34814 // cprintf (("sweep would be: %d, total_count: %d, ratio would be %d%% > target\n",
34815 // (sweep_count + 1), (total_count + 1), temp_ratio));
34816 should_compact = TRUE;
34821 return !should_compact;
34823 #endif //GC_CONFIG_DRIVEN
34825 void gc_heap::do_post_gc()
34827 if (!settings.concurrent)
34833 #ifdef COUNT_CYCLES
34834 AllocStart = GetCycleCount32();
34836 AllocStart = clock();
34837 #endif //COUNT_CYCLES
34840 #ifdef MULTIPLE_HEAPS
34841 gc_heap* hp = g_heaps[0];
34844 #endif //MULTIPLE_HEAPS
34846 GCToEEInterface::GcDone(settings.condemned_generation);
34848 GCToEEInterface::DiagGCEnd(VolatileLoad(&settings.gc_index),
34849 (uint32_t)settings.condemned_generation,
34850 (uint32_t)settings.reason,
34851 !!settings.concurrent);
34853 //dprintf (1, (" ****end of Garbage Collection**** %d(gen0:%d)(%d)",
34854 dprintf (1, ("*EGC* %d(gen0:%d)(%d)(%s)",
34855 VolatileLoad(&settings.gc_index),
34856 dd_collection_count(hp->dynamic_data_of(0)),
34857 settings.condemned_generation,
34858 (settings.concurrent ? "BGC" : "GC")));
34860 if (settings.exit_memory_load != 0)
34861 last_gc_memory_load = settings.exit_memory_load;
34862 else if (settings.entry_memory_load != 0)
34863 last_gc_memory_load = settings.entry_memory_load;
34865 last_gc_heap_size = get_total_heap_size();
34866 last_gc_fragmentation = get_total_fragmentation();
34868 GCHeap::UpdatePostGCCounters();
34869 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34870 //if (g_fEnableARM)
34872 // SystemDomain::GetADSurvivedBytes();
34874 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34877 STRESS_LOG_GC_END(VolatileLoad(&settings.gc_index),
34878 (uint32_t)settings.condemned_generation,
34879 (uint32_t)settings.reason);
34880 #endif // STRESS_LOG
34882 #ifdef GC_CONFIG_DRIVEN
34883 if (!settings.concurrent)
34885 if (settings.compaction)
34886 (compact_or_sweep_gcs[0])++;
34888 (compact_or_sweep_gcs[1])++;
34891 #ifdef MULTIPLE_HEAPS
34892 for (int i = 0; i < n_heaps; i++)
34893 g_heaps[i]->record_interesting_info_per_heap();
34895 record_interesting_info_per_heap();
34896 #endif //MULTIPLE_HEAPS
34897 record_global_mechanisms();
34898 #endif //GC_CONFIG_DRIVEN
34901 unsigned GCHeap::GetGcCount()
34903 return (unsigned int)VolatileLoad(&pGenGCHeap->settings.gc_index);
34907 GCHeap::GarbageCollectGeneration (unsigned int gen, gc_reason reason)
34909 dprintf (2, ("triggered a GC!"));
34911 #ifdef MULTIPLE_HEAPS
34912 gc_heap* hpt = gc_heap::g_heaps[0];
34915 #endif //MULTIPLE_HEAPS
34916 bool cooperative_mode = true;
34917 dynamic_data* dd = hpt->dynamic_data_of (gen);
34918 size_t localCount = dd_collection_count (dd);
34920 enter_spin_lock (&gc_heap::gc_lock);
34921 dprintf (SPINLOCK_LOG, ("GC Egc"));
34922 ASSERT_HOLDING_SPIN_LOCK(&gc_heap::gc_lock);
34924 //don't trigger another GC if one was already in progress
34925 //while waiting for the lock
34927 size_t col_count = dd_collection_count (dd);
34929 if (localCount != col_count)
34931 #ifdef SYNCHRONIZATION_STATS
34932 gc_lock_contended++;
34933 #endif //SYNCHRONIZATION_STATS
34934 dprintf (SPINLOCK_LOG, ("no need GC Lgc"));
34935 leave_spin_lock (&gc_heap::gc_lock);
34937 // We don't need to release msl here 'cause this means a GC
34938 // has happened and would have release all msl's.
34943 #ifdef COUNT_CYCLES
34944 int gc_start = GetCycleCount32();
34945 #endif //COUNT_CYCLES
34948 #ifdef COUNT_CYCLES
34949 AllocDuration += GetCycleCount32() - AllocStart;
34951 AllocDuration += clock() - AllocStart;
34952 #endif //COUNT_CYCLES
34955 gc_heap::g_low_memory_status = (reason == reason_lowmemory) ||
34956 (reason == reason_lowmemory_blocking) ||
34957 (gc_heap::latency_level == latency_level_memory_footprint);
34959 gc_trigger_reason = reason;
34961 #ifdef MULTIPLE_HEAPS
34962 for (int i = 0; i < gc_heap::n_heaps; i++)
34964 gc_heap::g_heaps[i]->reset_gc_done();
34967 gc_heap::reset_gc_done();
34968 #endif //MULTIPLE_HEAPS
34970 gc_heap::gc_started = TRUE;
34973 init_sync_log_stats();
34975 #ifndef MULTIPLE_HEAPS
34976 cooperative_mode = gc_heap::enable_preemptive ();
34978 dprintf (2, ("Suspending EE"));
34979 BEGIN_TIMING(suspend_ee_during_log);
34980 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
34981 END_TIMING(suspend_ee_during_log);
34982 gc_heap::proceed_with_gc_p = gc_heap::should_proceed_with_gc();
34983 gc_heap::disable_preemptive (cooperative_mode);
34984 if (gc_heap::proceed_with_gc_p)
34985 pGenGCHeap->settings.init_mechanisms();
34987 gc_heap::update_collection_counts_for_no_gc();
34989 #endif //!MULTIPLE_HEAPS
34992 // MAP_EVENT_MONITORS(EE_MONITOR_GARBAGE_COLLECTIONS, NotifyEvent(EE_EVENT_TYPE_GC_STARTED, 0));
34995 #ifdef COUNT_CYCLES
34998 start = GetCycleCount32();
35003 #endif //COUNT_CYCLES
35004 PromotedObjectCount = 0;
35007 unsigned int condemned_generation_number = gen;
35009 // We want to get a stack from the user thread that triggered the GC
35010 // instead of on the GC thread which is the case for Server GC.
35011 // But we are doing it for Workstation GC as well to be uniform.
35012 FIRE_EVENT(GCTriggered, static_cast<uint32_t>(reason));
35014 #ifdef MULTIPLE_HEAPS
35015 GcCondemnedGeneration = condemned_generation_number;
35017 cooperative_mode = gc_heap::enable_preemptive ();
35019 BEGIN_TIMING(gc_during_log);
35020 gc_heap::ee_suspend_event.Set();
35021 gc_heap::wait_for_gc_done();
35022 END_TIMING(gc_during_log);
35024 gc_heap::disable_preemptive (cooperative_mode);
35026 condemned_generation_number = GcCondemnedGeneration;
35028 if (gc_heap::proceed_with_gc_p)
35030 BEGIN_TIMING(gc_during_log);
35031 pGenGCHeap->garbage_collect (condemned_generation_number);
35032 END_TIMING(gc_during_log);
35034 #endif //MULTIPLE_HEAPS
35037 #ifdef COUNT_CYCLES
35038 finish = GetCycleCount32();
35041 #endif //COUNT_CYCLES
35042 GcDuration += finish - start;
35044 ("<GC# %d> Condemned: %d, Duration: %d, total: %d Alloc Avg: %d, Small Objects:%d Large Objects:%d",
35045 VolatileLoad(&pGenGCHeap->settings.gc_index), condemned_generation_number,
35046 finish - start, GcDuration,
35047 AllocCount ? (AllocDuration / AllocCount) : 0,
35048 AllocSmallCount, AllocBigCount));
35053 #ifdef BACKGROUND_GC
35054 // We are deciding whether we should fire the alloc wait end event here
35055 // because in begin_foreground we could be calling end_foreground
35056 // if we need to retry.
35057 if (gc_heap::alloc_wait_event_p)
35059 hpt->fire_alloc_wait_event_end (awr_fgc_wait_for_bgc);
35060 gc_heap::alloc_wait_event_p = FALSE;
35062 #endif //BACKGROUND_GC
35064 #ifndef MULTIPLE_HEAPS
35065 #ifdef BACKGROUND_GC
35066 if (!gc_heap::dont_restart_ee_p)
35068 #endif //BACKGROUND_GC
35069 BEGIN_TIMING(restart_ee_during_log);
35070 GCToEEInterface::RestartEE(TRUE);
35071 END_TIMING(restart_ee_during_log);
35072 #ifdef BACKGROUND_GC
35074 #endif //BACKGROUND_GC
35075 #endif //!MULTIPLE_HEAPS
35077 #ifdef COUNT_CYCLES
35078 printf ("GC: %d Time: %d\n", GcCondemnedGeneration,
35079 GetCycleCount32() - gc_start);
35080 #endif //COUNT_CYCLES
35082 #ifndef MULTIPLE_HEAPS
35083 process_sync_log_stats();
35084 gc_heap::gc_started = FALSE;
35085 gc_heap::set_gc_done();
35086 dprintf (SPINLOCK_LOG, ("GC Lgc"));
35087 leave_spin_lock (&gc_heap::gc_lock);
35088 #endif //!MULTIPLE_HEAPS
35090 #ifdef FEATURE_PREMORTEM_FINALIZATION
35091 GCToEEInterface::EnableFinalization(!pGenGCHeap->settings.concurrent && pGenGCHeap->settings.found_finalizers);
35092 #endif // FEATURE_PREMORTEM_FINALIZATION
35094 return dd_collection_count (dd);
35097 size_t GCHeap::GetTotalBytesInUse ()
35099 #ifdef MULTIPLE_HEAPS
35100 //enumarate all the heaps and get their size.
35101 size_t tot_size = 0;
35102 for (int i = 0; i < gc_heap::n_heaps; i++)
35104 GCHeap* Hp = gc_heap::g_heaps [i]->vm_heap;
35105 tot_size += Hp->ApproxTotalBytesInUse (FALSE);
35109 return ApproxTotalBytesInUse ();
35110 #endif //MULTIPLE_HEAPS
35113 int GCHeap::CollectionCount (int generation, int get_bgc_fgc_count)
35115 if (get_bgc_fgc_count != 0)
35117 #ifdef BACKGROUND_GC
35118 if (generation == max_generation)
35120 return (int)(gc_heap::full_gc_counts[gc_type_background]);
35124 return (int)(gc_heap::ephemeral_fgc_counts[generation]);
35128 #endif //BACKGROUND_GC
35131 #ifdef MULTIPLE_HEAPS
35132 gc_heap* hp = gc_heap::g_heaps [0];
35133 #else //MULTIPLE_HEAPS
35134 gc_heap* hp = pGenGCHeap;
35135 #endif //MULTIPLE_HEAPS
35136 if (generation > max_generation)
35139 return (int)dd_collection_count (hp->dynamic_data_of (generation));
35142 size_t GCHeap::ApproxTotalBytesInUse(BOOL small_heap_only)
35144 size_t totsize = 0;
35146 //ASSERT(InMustComplete());
35147 enter_spin_lock (&pGenGCHeap->gc_lock);
35149 heap_segment* eph_seg = generation_allocation_segment (pGenGCHeap->generation_of (0));
35150 // Get small block heap size info
35151 totsize = (pGenGCHeap->alloc_allocated - heap_segment_mem (eph_seg));
35152 heap_segment* seg1 = generation_start_segment (pGenGCHeap->generation_of (max_generation));
35153 while (seg1 != eph_seg)
35155 totsize += heap_segment_allocated (seg1) -
35156 heap_segment_mem (seg1);
35157 seg1 = heap_segment_next (seg1);
35160 //discount the fragmentation
35161 for (int i = 0; i <= max_generation; i++)
35163 generation* gen = pGenGCHeap->generation_of (i);
35164 totsize -= (generation_free_list_space (gen) + generation_free_obj_space (gen));
35167 if (!small_heap_only)
35169 heap_segment* seg2 = generation_start_segment (pGenGCHeap->generation_of (max_generation+1));
35173 totsize += heap_segment_allocated (seg2) -
35174 heap_segment_mem (seg2);
35175 seg2 = heap_segment_next (seg2);
35178 //discount the fragmentation
35179 generation* loh_gen = pGenGCHeap->generation_of (max_generation+1);
35180 size_t frag = generation_free_list_space (loh_gen) + generation_free_obj_space (loh_gen);
35183 leave_spin_lock (&pGenGCHeap->gc_lock);
35187 #ifdef MULTIPLE_HEAPS
35188 void GCHeap::AssignHeap (alloc_context* acontext)
35190 // Assign heap based on processor
35191 acontext->set_alloc_heap(GetHeap(heap_select::select_heap(acontext, 0)));
35192 acontext->set_home_heap(acontext->get_alloc_heap());
35194 GCHeap* GCHeap::GetHeap (int n)
35196 assert (n < gc_heap::n_heaps);
35197 return gc_heap::g_heaps [n]->vm_heap;
35199 #endif //MULTIPLE_HEAPS
35201 bool GCHeap::IsThreadUsingAllocationContextHeap(gc_alloc_context* context, int thread_number)
35203 alloc_context* acontext = static_cast<alloc_context*>(context);
35204 #ifdef MULTIPLE_HEAPS
35205 return ((acontext->get_home_heap() == GetHeap(thread_number)) ||
35206 ((acontext->get_home_heap() == 0) && (thread_number == 0)));
35208 UNREFERENCED_PARAMETER(acontext);
35209 UNREFERENCED_PARAMETER(thread_number);
35211 #endif //MULTIPLE_HEAPS
35214 // Returns the number of processors required to trigger the use of thread based allocation contexts
35215 int GCHeap::GetNumberOfHeaps ()
35217 #ifdef MULTIPLE_HEAPS
35218 return gc_heap::n_heaps;
35221 #endif //MULTIPLE_HEAPS
35225 in this way we spend extra time cycling through all the heaps while create the handle
35226 it ought to be changed by keeping alloc_context.home_heap as number (equals heap_number)
35228 int GCHeap::GetHomeHeapNumber ()
35230 #ifdef MULTIPLE_HEAPS
35231 Thread *pThread = GCToEEInterface::GetThread();
35232 for (int i = 0; i < gc_heap::n_heaps; i++)
35236 gc_alloc_context* ctx = GCToEEInterface::GetAllocContext();
35237 GCHeap *hp = static_cast<alloc_context*>(ctx)->get_home_heap();
35238 if (hp == gc_heap::g_heaps[i]->vm_heap) return i;
35244 #endif //MULTIPLE_HEAPS
35247 unsigned int GCHeap::GetCondemnedGeneration()
35249 return gc_heap::settings.condemned_generation;
35252 void GCHeap::GetMemoryInfo(uint32_t* highMemLoadThreshold,
35253 uint64_t* totalPhysicalMem,
35254 uint32_t* lastRecordedMemLoad,
35255 size_t* lastRecordedHeapSize,
35256 size_t* lastRecordedFragmentation)
35258 *highMemLoadThreshold = gc_heap::high_memory_load_th;
35259 *totalPhysicalMem = gc_heap::total_physical_mem;
35260 *lastRecordedMemLoad = gc_heap::last_gc_memory_load;
35261 *lastRecordedHeapSize = gc_heap::last_gc_heap_size;
35262 *lastRecordedFragmentation = gc_heap::last_gc_fragmentation;
35265 int GCHeap::GetGcLatencyMode()
35267 return (int)(pGenGCHeap->settings.pause_mode);
35270 int GCHeap::SetGcLatencyMode (int newLatencyMode)
35272 if (gc_heap::settings.pause_mode == pause_no_gc)
35273 return (int)set_pause_mode_no_gc;
35275 gc_pause_mode new_mode = (gc_pause_mode)newLatencyMode;
35277 if (new_mode == pause_low_latency)
35279 #ifndef MULTIPLE_HEAPS
35280 pGenGCHeap->settings.pause_mode = new_mode;
35281 #endif //!MULTIPLE_HEAPS
35283 else if (new_mode == pause_sustained_low_latency)
35285 #ifdef BACKGROUND_GC
35286 if (gc_heap::gc_can_use_concurrent)
35288 pGenGCHeap->settings.pause_mode = new_mode;
35290 #endif //BACKGROUND_GC
35294 pGenGCHeap->settings.pause_mode = new_mode;
35297 #ifdef BACKGROUND_GC
35298 if (recursive_gc_sync::background_running_p())
35300 // If we get here, it means we are doing an FGC. If the pause
35301 // mode was altered we will need to save it in the BGC settings.
35302 if (gc_heap::saved_bgc_settings.pause_mode != new_mode)
35304 gc_heap::saved_bgc_settings.pause_mode = new_mode;
35307 #endif //BACKGROUND_GC
35309 return (int)set_pause_mode_success;
35312 int GCHeap::GetLOHCompactionMode()
35314 return pGenGCHeap->loh_compaction_mode;
35317 void GCHeap::SetLOHCompactionMode (int newLOHCompactionyMode)
35319 #ifdef FEATURE_LOH_COMPACTION
35320 pGenGCHeap->loh_compaction_mode = (gc_loh_compaction_mode)newLOHCompactionyMode;
35321 #endif //FEATURE_LOH_COMPACTION
35324 bool GCHeap::RegisterForFullGCNotification(uint32_t gen2Percentage,
35325 uint32_t lohPercentage)
35327 #ifdef MULTIPLE_HEAPS
35328 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35330 gc_heap* hp = gc_heap::g_heaps [hn];
35331 hp->fgn_last_alloc = dd_new_allocation (hp->dynamic_data_of (0));
35333 #else //MULTIPLE_HEAPS
35334 pGenGCHeap->fgn_last_alloc = dd_new_allocation (pGenGCHeap->dynamic_data_of (0));
35335 #endif //MULTIPLE_HEAPS
35337 pGenGCHeap->full_gc_approach_event.Reset();
35338 pGenGCHeap->full_gc_end_event.Reset();
35339 pGenGCHeap->full_gc_approach_event_set = false;
35341 pGenGCHeap->fgn_maxgen_percent = gen2Percentage;
35342 pGenGCHeap->fgn_loh_percent = lohPercentage;
35347 bool GCHeap::CancelFullGCNotification()
35349 pGenGCHeap->fgn_maxgen_percent = 0;
35350 pGenGCHeap->fgn_loh_percent = 0;
35352 pGenGCHeap->full_gc_approach_event.Set();
35353 pGenGCHeap->full_gc_end_event.Set();
35358 int GCHeap::WaitForFullGCApproach(int millisecondsTimeout)
35360 dprintf (2, ("WFGA: Begin wait"));
35361 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_approach_event), millisecondsTimeout);
35362 dprintf (2, ("WFGA: End wait"));
35366 int GCHeap::WaitForFullGCComplete(int millisecondsTimeout)
35368 dprintf (2, ("WFGE: Begin wait"));
35369 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_end_event), millisecondsTimeout);
35370 dprintf (2, ("WFGE: End wait"));
35374 int GCHeap::StartNoGCRegion(uint64_t totalSize, bool lohSizeKnown, uint64_t lohSize, bool disallowFullBlockingGC)
35376 NoGCRegionLockHolder lh;
35378 dprintf (1, ("begin no gc called"));
35379 start_no_gc_region_status status = gc_heap::prepare_for_no_gc_region (totalSize, lohSizeKnown, lohSize, disallowFullBlockingGC);
35380 if (status == start_no_gc_success)
35382 GarbageCollect (max_generation);
35383 status = gc_heap::get_start_no_gc_region_status();
35386 if (status != start_no_gc_success)
35387 gc_heap::handle_failure_for_no_gc();
35389 return (int)status;
35392 int GCHeap::EndNoGCRegion()
35394 NoGCRegionLockHolder lh;
35395 return (int)gc_heap::end_no_gc_region();
35398 void GCHeap::PublishObject (uint8_t* Obj)
35400 #ifdef BACKGROUND_GC
35401 gc_heap* hp = gc_heap::heap_of (Obj);
35402 hp->bgc_alloc_lock->loh_alloc_done (Obj);
35403 #endif //BACKGROUND_GC
35406 // The spec for this one isn't clear. This function
35407 // returns the size that can be allocated without
35408 // triggering a GC of any kind.
35409 size_t GCHeap::ApproxFreeBytes()
35412 //ASSERT(InMustComplete());
35413 enter_spin_lock (&pGenGCHeap->gc_lock);
35415 generation* gen = pGenGCHeap->generation_of (0);
35416 size_t res = generation_allocation_limit (gen) - generation_allocation_pointer (gen);
35418 leave_spin_lock (&pGenGCHeap->gc_lock);
35423 HRESULT GCHeap::GetGcCounters(int gen, gc_counters* counters)
35425 if ((gen < 0) || (gen > max_generation))
35427 #ifdef MULTIPLE_HEAPS
35428 counters->current_size = 0;
35429 counters->promoted_size = 0;
35430 counters->collection_count = 0;
35432 //enumarate all the heaps and get their counters.
35433 for (int i = 0; i < gc_heap::n_heaps; i++)
35435 dynamic_data* dd = gc_heap::g_heaps [i]->dynamic_data_of (gen);
35437 counters->current_size += dd_current_size (dd);
35438 counters->promoted_size += dd_promoted_size (dd);
35440 counters->collection_count += dd_collection_count (dd);
35443 dynamic_data* dd = pGenGCHeap->dynamic_data_of (gen);
35444 counters->current_size = dd_current_size (dd);
35445 counters->promoted_size = dd_promoted_size (dd);
35446 counters->collection_count = dd_collection_count (dd);
35447 #endif //MULTIPLE_HEAPS
35451 // Get the segment size to use, making sure it conforms.
35452 size_t GCHeap::GetValidSegmentSize(bool large_seg)
35454 return get_valid_segment_size (large_seg);
35457 // Get the max gen0 heap size, making sure it conforms.
35458 size_t GCHeap::GetValidGen0MaxSize(size_t seg_size)
35460 size_t gen0size = static_cast<size_t>(GCConfig::GetGen0Size());
35462 if ((gen0size == 0) || !g_theGCHeap->IsValidGen0MaxSize(gen0size))
35465 // performance data seems to indicate halving the size results
35466 // in optimal perf. Ask for adjusted gen0 size.
35467 gen0size = max(GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),(256*1024));
35469 // if gen0 size is too large given the available memory, reduce it.
35470 // Get true cache size, as we don't want to reduce below this.
35471 size_t trueSize = max(GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE),(256*1024));
35472 dprintf (2, ("cache: %Id-%Id, cpu: %Id",
35473 GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),
35474 GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)));
35476 int n_heaps = gc_heap::n_heaps;
35478 size_t trueSize = GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE);
35479 gen0size = max((4*trueSize/5),(256*1024));
35480 trueSize = max(trueSize, (256*1024));
35484 // if the total min GC across heaps will exceed 1/6th of available memory,
35485 // then reduce the min GC size until it either fits or has been reduced to cache size.
35486 while ((gen0size * n_heaps) > GCToOSInterface::GetPhysicalMemoryLimit() / 6)
35488 gen0size = gen0size / 2;
35489 if (gen0size <= trueSize)
35491 gen0size = trueSize;
35497 // Generation 0 must never be more than 1/2 the segment size.
35498 if (gen0size >= (seg_size / 2))
35499 gen0size = seg_size / 2;
35504 void GCHeap::SetReservedVMLimit (size_t vmlimit)
35506 gc_heap::reserved_memory_limit = vmlimit;
35510 //versions of same method on each heap
35512 #ifdef FEATURE_PREMORTEM_FINALIZATION
35514 Object* GCHeap::GetNextFinalizableObject()
35517 #ifdef MULTIPLE_HEAPS
35519 //return the first non critical one in the first queue.
35520 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35522 gc_heap* hp = gc_heap::g_heaps [hn];
35523 Object* O = hp->finalize_queue->GetNextFinalizableObject(TRUE);
35527 //return the first non crtitical/critical one in the first queue.
35528 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35530 gc_heap* hp = gc_heap::g_heaps [hn];
35531 Object* O = hp->finalize_queue->GetNextFinalizableObject(FALSE);
35538 #else //MULTIPLE_HEAPS
35539 return pGenGCHeap->finalize_queue->GetNextFinalizableObject();
35540 #endif //MULTIPLE_HEAPS
35544 size_t GCHeap::GetNumberFinalizableObjects()
35546 #ifdef MULTIPLE_HEAPS
35548 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35550 gc_heap* hp = gc_heap::g_heaps [hn];
35551 cnt += hp->finalize_queue->GetNumberFinalizableObjects();
35556 #else //MULTIPLE_HEAPS
35557 return pGenGCHeap->finalize_queue->GetNumberFinalizableObjects();
35558 #endif //MULTIPLE_HEAPS
35561 size_t GCHeap::GetFinalizablePromotedCount()
35563 #ifdef MULTIPLE_HEAPS
35566 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35568 gc_heap* hp = gc_heap::g_heaps [hn];
35569 cnt += hp->finalize_queue->GetPromotedCount();
35573 #else //MULTIPLE_HEAPS
35574 return pGenGCHeap->finalize_queue->GetPromotedCount();
35575 #endif //MULTIPLE_HEAPS
35578 bool GCHeap::FinalizeAppDomain(void *pDomain, bool fRunFinalizers)
35580 #ifdef MULTIPLE_HEAPS
35581 bool foundp = false;
35582 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35584 gc_heap* hp = gc_heap::g_heaps [hn];
35585 if (hp->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers))
35590 #else //MULTIPLE_HEAPS
35591 return pGenGCHeap->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers);
35592 #endif //MULTIPLE_HEAPS
35595 bool GCHeap::ShouldRestartFinalizerWatchDog()
35597 // This condition was historically used as part of the condition to detect finalizer thread timeouts
35598 return gc_heap::gc_lock.lock != -1;
35601 void GCHeap::SetFinalizeQueueForShutdown(bool fHasLock)
35603 #ifdef MULTIPLE_HEAPS
35604 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35606 gc_heap* hp = gc_heap::g_heaps [hn];
35607 hp->finalize_queue->SetSegForShutDown(fHasLock);
35610 #else //MULTIPLE_HEAPS
35611 pGenGCHeap->finalize_queue->SetSegForShutDown(fHasLock);
35612 #endif //MULTIPLE_HEAPS
35615 //---------------------------------------------------------------------------
35616 // Finalized class tracking
35617 //---------------------------------------------------------------------------
35619 bool GCHeap::RegisterForFinalization (int gen, Object* obj)
35623 if (((((CObjectHeader*)obj)->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN))
35625 //just reset the bit
35626 ((CObjectHeader*)obj)->GetHeader()->ClrBit(BIT_SBLK_FINALIZER_RUN);
35631 gc_heap* hp = gc_heap::heap_of ((uint8_t*)obj);
35632 return hp->finalize_queue->RegisterForFinalization (gen, obj);
35636 void GCHeap::SetFinalizationRun (Object* obj)
35638 ((CObjectHeader*)obj)->GetHeader()->SetBit(BIT_SBLK_FINALIZER_RUN);
35642 //--------------------------------------------------------------------
35644 // Support for finalization
35646 //--------------------------------------------------------------------
35649 unsigned int gen_segment (int gen)
35651 assert (((signed)NUMBERGENERATIONS - gen - 1)>=0);
35652 return (NUMBERGENERATIONS - gen - 1);
35655 bool CFinalize::Initialize()
35662 m_Array = new (nothrow)(Object*[100]);
35667 STRESS_LOG_OOM_STACK(sizeof(Object*[100]));
35668 if (GCConfig::GetBreakOnOOM())
35670 GCToOSInterface::DebugBreak();
35674 m_EndArray = &m_Array[100];
35676 for (int i =0; i < FreeList; i++)
35678 SegQueueLimit (i) = m_Array;
35680 m_PromotedCount = 0;
35683 lockowner_threadid.Clear();
35689 CFinalize::~CFinalize()
35694 size_t CFinalize::GetPromotedCount ()
35696 return m_PromotedCount;
35700 void CFinalize::EnterFinalizeLock()
35702 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
35703 GCToEEInterface::GetThread() == 0 ||
35704 GCToEEInterface::IsPreemptiveGCDisabled());
35707 if (Interlocked::CompareExchange(&lock, 0, -1) >= 0)
35709 unsigned int i = 0;
35712 YieldProcessor(); // indicate to the processor that we are spining
35714 GCToOSInterface::YieldThread (0);
35716 GCToOSInterface::Sleep (5);
35722 lockowner_threadid.SetToCurrentThread();
35727 void CFinalize::LeaveFinalizeLock()
35729 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
35730 GCToEEInterface::GetThread() == 0 ||
35731 GCToEEInterface::IsPreemptiveGCDisabled());
35734 lockowner_threadid.Clear();
35740 CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size)
35747 EnterFinalizeLock();
35749 unsigned int dest = 0;
35751 if (g_fFinalizerRunOnShutDown)
35753 //no method table available yet,
35754 //put it in the finalizer queue and sort out when
35756 dest = FinalizerListSeg;
35760 dest = gen_segment (gen);
35762 // Adjust boundary for segments so that GC will keep objects alive.
35763 Object*** s_i = &SegQueue (FreeList);
35764 if ((*s_i) == m_EndArray)
35768 LeaveFinalizeLock();
35769 if (method_table(obj) == NULL)
35771 // If the object is uninitialized, a valid size should have been passed.
35772 assert (size >= Align (min_obj_size));
35773 dprintf (3, ("Making unused array [%Ix, %Ix[", (size_t)obj, (size_t)(obj+size)));
35774 ((CObjectHeader*)obj)->SetFree(size);
35776 STRESS_LOG_OOM_STACK(0);
35777 if (GCConfig::GetBreakOnOOM())
35779 GCToOSInterface::DebugBreak();
35784 Object*** end_si = &SegQueueLimit (dest);
35787 //is the segment empty?
35788 if (!(*s_i == *(s_i-1)))
35790 //no, swap the end elements.
35791 *(*s_i) = *(*(s_i-1));
35793 //increment the fill pointer
35795 //go to the next segment.
35797 } while (s_i > end_si);
35799 // We have reached the destination segment
35800 // store the object
35802 // increment the fill pointer
35805 LeaveFinalizeLock();
35811 CFinalize::GetNextFinalizableObject (BOOL only_non_critical)
35815 EnterFinalizeLock();
35818 if (!IsSegEmpty(FinalizerListSeg))
35820 if (g_fFinalizerRunOnShutDown)
35822 obj = *(SegQueueLimit (FinalizerListSeg)-1);
35823 if (method_table(obj)->HasCriticalFinalizer())
35825 MoveItem ((SegQueueLimit (FinalizerListSeg)-1),
35826 FinalizerListSeg, CriticalFinalizerListSeg);
35830 --SegQueueLimit (FinalizerListSeg);
35833 obj = *(--SegQueueLimit (FinalizerListSeg));
35836 else if (!only_non_critical && !IsSegEmpty(CriticalFinalizerListSeg))
35838 //the FinalizerList is empty, we can adjust both
35839 // limit instead of moving the object to the free list
35840 obj = *(--SegQueueLimit (CriticalFinalizerListSeg));
35841 --SegQueueLimit (FinalizerListSeg);
35845 dprintf (3, ("running finalizer for %Ix (mt: %Ix)", obj, method_table (obj)));
35847 LeaveFinalizeLock();
35852 CFinalize::SetSegForShutDown(BOOL fHasLock)
35857 EnterFinalizeLock();
35858 for (i = 0; i <= max_generation; i++)
35860 unsigned int seg = gen_segment (i);
35861 Object** startIndex = SegQueueLimit (seg)-1;
35862 Object** stopIndex = SegQueue (seg);
35863 for (Object** po = startIndex; po >= stopIndex; po--)
35866 if (method_table(obj)->HasCriticalFinalizer())
35868 MoveItem (po, seg, CriticalFinalizerListSeg);
35872 MoveItem (po, seg, FinalizerListSeg);
35877 LeaveFinalizeLock();
35881 CFinalize::DiscardNonCriticalObjects()
35883 //empty the finalization queue
35884 Object** startIndex = SegQueueLimit (FinalizerListSeg)-1;
35885 Object** stopIndex = SegQueue (FinalizerListSeg);
35886 for (Object** po = startIndex; po >= stopIndex; po--)
35888 MoveItem (po, FinalizerListSeg, FreeList);
35893 CFinalize::GetNumberFinalizableObjects()
35895 return SegQueueLimit (FinalizerListSeg) -
35896 (g_fFinalizerRunOnShutDown ? m_Array : SegQueue(FinalizerListSeg));
35900 CFinalize::FinalizeSegForAppDomain (void *pDomain,
35901 BOOL fRunFinalizers,
35904 BOOL finalizedFound = FALSE;
35905 Object** endIndex = SegQueue (Seg);
35906 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
35908 CObjectHeader* obj = (CObjectHeader*)*i;
35910 // Objects are put into the finalization queue before they are complete (ie their methodtable
35911 // may be null) so we must check that the object we found has a method table before checking
35912 // if it has the index we are looking for. If the methodtable is null, it can't be from the
35913 // unloading domain, so skip it.
35914 if (method_table(obj) == NULL)
35919 // does the EE actually want us to finalize this object?
35920 if (!GCToEEInterface::ShouldFinalizeObjectForUnload(pDomain, obj))
35925 if (!fRunFinalizers || (obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
35927 //remove the object because we don't want to
35928 //run the finalizer
35929 MoveItem (i, Seg, FreeList);
35930 //Reset the bit so it will be put back on the queue
35931 //if resurrected and re-registered.
35932 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
35936 if (method_table(obj)->HasCriticalFinalizer())
35938 finalizedFound = TRUE;
35939 MoveItem (i, Seg, CriticalFinalizerListSeg);
35943 if (GCToEEInterface::AppDomainIsRudeUnload(pDomain))
35945 MoveItem (i, Seg, FreeList);
35949 finalizedFound = TRUE;
35950 MoveItem (i, Seg, FinalizerListSeg);
35956 return finalizedFound;
35960 CFinalize::FinalizeAppDomain (void *pDomain, bool fRunFinalizers)
35962 bool finalizedFound = false;
35964 unsigned int startSeg = gen_segment (max_generation);
35966 EnterFinalizeLock();
35968 for (unsigned int Seg = startSeg; Seg <= gen_segment (0); Seg++)
35970 if (FinalizeSegForAppDomain (pDomain, fRunFinalizers, Seg))
35972 finalizedFound = true;
35976 LeaveFinalizeLock();
35978 return finalizedFound;
35982 CFinalize::MoveItem (Object** fromIndex,
35983 unsigned int fromSeg,
35984 unsigned int toSeg)
35988 ASSERT (fromSeg != toSeg);
35989 if (fromSeg > toSeg)
35993 // Place the element at the boundary closest to dest
35994 Object** srcIndex = fromIndex;
35995 for (unsigned int i = fromSeg; i != toSeg; i+= step)
35997 Object**& destFill = m_FillPointers[i+(step - 1 )/2];
35998 Object** destIndex = destFill - (step + 1)/2;
35999 if (srcIndex != destIndex)
36001 Object* tmp = *srcIndex;
36002 *srcIndex = *destIndex;
36006 srcIndex = destIndex;
36011 CFinalize::GcScanRoots (promote_func* fn, int hn, ScanContext *pSC)
36017 pSC->thread_number = hn;
36019 //scan the finalization queue
36020 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
36021 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
36023 for (Object** po = startIndex; po < stopIndex; po++)
36026 //dprintf (3, ("scan freacheable %Ix", (size_t)o));
36027 dprintf (3, ("scan f %Ix", (size_t)o));
36028 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
36031 pSC->pCurrentDomain = SystemDomain::GetAppDomainAtIndex(o->GetAppDomainIndex());
36033 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
36039 void CFinalize::WalkFReachableObjects (fq_walk_fn fn)
36041 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
36042 Object** stopCriticalIndex = SegQueueLimit (CriticalFinalizerListSeg);
36043 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
36044 for (Object** po = startIndex; po < stopIndex; po++)
36047 fn(po < stopCriticalIndex, *po);
36052 CFinalize::ScanForFinalization (promote_func* pfn, int gen, BOOL mark_only_p,
36056 sc.promotion = TRUE;
36057 #ifdef MULTIPLE_HEAPS
36058 sc.thread_number = hp->heap_number;
36060 UNREFERENCED_PARAMETER(hp);
36061 #endif //MULTIPLE_HEAPS
36063 BOOL finalizedFound = FALSE;
36065 //start with gen and explore all the younger generations.
36066 unsigned int startSeg = gen_segment (gen);
36068 m_PromotedCount = 0;
36069 for (unsigned int Seg = startSeg; Seg <= gen_segment(0); Seg++)
36071 Object** endIndex = SegQueue (Seg);
36072 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
36074 CObjectHeader* obj = (CObjectHeader*)*i;
36075 dprintf (3, ("scanning: %Ix", (size_t)obj));
36076 if (!g_theGCHeap->IsPromoted (obj))
36078 dprintf (3, ("freacheable: %Ix", (size_t)obj));
36080 assert (method_table(obj)->HasFinalizer());
36082 if (GCToEEInterface::EagerFinalized(obj))
36084 MoveItem (i, Seg, FreeList);
36086 else if ((obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
36088 //remove the object because we don't want to
36089 //run the finalizer
36091 MoveItem (i, Seg, FreeList);
36093 //Reset the bit so it will be put back on the queue
36094 //if resurrected and re-registered.
36095 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
36102 if (method_table(obj)->HasCriticalFinalizer())
36104 MoveItem (i, Seg, CriticalFinalizerListSeg);
36108 MoveItem (i, Seg, FinalizerListSeg);
36112 #ifdef BACKGROUND_GC
36115 if ((gen == max_generation) && (recursive_gc_sync::background_running_p()))
36117 // TODO - fix the following line.
36118 //assert (gc_heap::background_object_marked ((uint8_t*)obj, FALSE));
36119 dprintf (3, ("%Ix is marked", (size_t)obj));
36122 #endif //BACKGROUND_GC
36126 finalizedFound = !IsSegEmpty(FinalizerListSeg) ||
36127 !IsSegEmpty(CriticalFinalizerListSeg);
36129 if (finalizedFound)
36131 //Promote the f-reachable objects
36133 #ifdef MULTIPLE_HEAPS
36137 #endif //MULTIPLE_HEAPS
36140 hp->settings.found_finalizers = TRUE;
36142 #ifdef BACKGROUND_GC
36143 if (hp->settings.concurrent)
36145 hp->settings.found_finalizers = !(IsSegEmpty(FinalizerListSeg) && IsSegEmpty(CriticalFinalizerListSeg));
36147 #endif //BACKGROUND_GC
36148 if (hp->settings.concurrent && hp->settings.found_finalizers)
36151 GCToEEInterface::EnableFinalization(true);
36155 return finalizedFound;
36158 //Relocates all of the objects in the finalization array
36160 CFinalize::RelocateFinalizationData (int gen, gc_heap* hp)
36163 sc.promotion = FALSE;
36164 #ifdef MULTIPLE_HEAPS
36165 sc.thread_number = hp->heap_number;
36167 UNREFERENCED_PARAMETER(hp);
36168 #endif //MULTIPLE_HEAPS
36170 unsigned int Seg = gen_segment (gen);
36172 Object** startIndex = SegQueue (Seg);
36173 for (Object** po = startIndex; po < SegQueue (FreeList);po++)
36175 GCHeap::Relocate (po, &sc);
36180 CFinalize::UpdatePromotedGenerations (int gen, BOOL gen_0_empty_p)
36182 // update the generation fill pointers.
36183 // if gen_0_empty is FALSE, test each object to find out if
36184 // it was promoted or not
36187 for (int i = min (gen+1, max_generation); i > 0; i--)
36189 m_FillPointers [gen_segment(i)] = m_FillPointers [gen_segment(i-1)];
36194 //Look for demoted or promoted plugs
36196 for (int i = gen; i >= 0; i--)
36198 unsigned int Seg = gen_segment (i);
36199 Object** startIndex = SegQueue (Seg);
36201 for (Object** po = startIndex;
36202 po < SegQueueLimit (gen_segment(i)); po++)
36204 int new_gen = g_theGCHeap->WhichGeneration (*po);
36210 MoveItem (po, gen_segment (i), gen_segment (new_gen));
36215 MoveItem (po, gen_segment (i), gen_segment (new_gen));
36216 //back down in order to see all objects.
36227 CFinalize::GrowArray()
36229 size_t oldArraySize = (m_EndArray - m_Array);
36230 size_t newArraySize = (size_t)(((float)oldArraySize / 10) * 12);
36232 Object** newArray = new (nothrow) Object*[newArraySize];
36235 // It's not safe to throw here, because of the FinalizeLock. Tell our caller
36236 // to throw for us.
36237 // ASSERT (newArray);
36240 memcpy (newArray, m_Array, oldArraySize*sizeof(Object*));
36242 //adjust the fill pointers
36243 for (int i = 0; i < FreeList; i++)
36245 m_FillPointers [i] += (newArray - m_Array);
36248 m_Array = newArray;
36249 m_EndArray = &m_Array [newArraySize];
36255 void CFinalize::CheckFinalizerObjects()
36257 for (int i = 0; i <= max_generation; i++)
36259 Object **startIndex = SegQueue (gen_segment (i));
36260 Object **stopIndex = SegQueueLimit (gen_segment (i));
36262 for (Object **po = startIndex; po < stopIndex; po++)
36264 if ((int)g_theGCHeap->WhichGeneration (*po) < i)
36266 ((CObjectHeader*)*po)->Validate();
36270 #endif //VERIFY_HEAP
36272 #endif // FEATURE_PREMORTEM_FINALIZATION
36275 //------------------------------------------------------------------------------
36277 // End of VM specific support
36279 //------------------------------------------------------------------------------
36280 void gc_heap::walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
36282 generation* gen = gc_heap::generation_of (gen_number);
36283 heap_segment* seg = generation_start_segment (gen);
36284 uint8_t* x = ((gen_number == max_generation) ? heap_segment_mem (seg) :
36285 generation_allocation_start (gen));
36287 uint8_t* end = heap_segment_allocated (seg);
36288 BOOL small_object_segments = TRUE;
36289 int align_const = get_alignment_constant (small_object_segments);
36296 if ((seg = heap_segment_next (seg)) != 0)
36298 x = heap_segment_mem (seg);
36299 end = heap_segment_allocated (seg);
36304 if (small_object_segments && walk_large_object_heap_p)
36307 small_object_segments = FALSE;
36308 align_const = get_alignment_constant (small_object_segments);
36309 seg = generation_start_segment (large_object_generation);
36310 x = heap_segment_mem (seg);
36311 end = heap_segment_allocated (seg);
36321 size_t s = size (x);
36322 CObjectHeader* o = (CObjectHeader*)x;
36327 _ASSERTE(((size_t)o & 0x3) == 0); // Last two bits should never be set at this point
36329 if (!fn (o->GetObjectBase(), context))
36332 x = x + Align (s, align_const);
36336 void gc_heap::walk_finalize_queue (fq_walk_fn fn)
36338 #ifdef FEATURE_PREMORTEM_FINALIZATION
36339 finalize_queue->WalkFReachableObjects (fn);
36340 #endif //FEATURE_PREMORTEM_FINALIZATION
36343 void gc_heap::walk_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
36345 #ifdef MULTIPLE_HEAPS
36346 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36348 gc_heap* hp = gc_heap::g_heaps [hn];
36350 hp->walk_heap_per_heap (fn, context, gen_number, walk_large_object_heap_p);
36353 walk_heap_per_heap(fn, context, gen_number, walk_large_object_heap_p);
36354 #endif //MULTIPLE_HEAPS
36357 void GCHeap::DiagWalkObject (Object* obj, walk_fn fn, void* context)
36359 uint8_t* o = (uint8_t*)obj;
36362 go_through_object_cl (method_table (o), o, size(o), oo,
36366 Object *oh = (Object*)*oo;
36367 if (!fn (oh, context))
36375 void GCHeap::DiagWalkSurvivorsWithType (void* gc_context, record_surv_fn fn, void* diag_context, walk_surv_type type)
36377 gc_heap* hp = (gc_heap*)gc_context;
36378 hp->walk_survivors (fn, diag_context, type);
36381 void GCHeap::DiagWalkHeap (walk_fn fn, void* context, int gen_number, bool walk_large_object_heap_p)
36383 gc_heap::walk_heap (fn, context, gen_number, walk_large_object_heap_p);
36386 void GCHeap::DiagWalkFinalizeQueue (void* gc_context, fq_walk_fn fn)
36388 gc_heap* hp = (gc_heap*)gc_context;
36389 hp->walk_finalize_queue (fn);
36392 void GCHeap::DiagScanFinalizeQueue (fq_scan_fn fn, ScanContext* sc)
36394 #ifdef MULTIPLE_HEAPS
36395 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36397 gc_heap* hp = gc_heap::g_heaps [hn];
36398 hp->finalize_queue->GcScanRoots(fn, hn, sc);
36401 pGenGCHeap->finalize_queue->GcScanRoots(fn, 0, sc);
36402 #endif //MULTIPLE_HEAPS
36405 void GCHeap::DiagScanHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
36407 UNREFERENCED_PARAMETER(gen_number);
36408 GCScan::GcScanHandlesForProfilerAndETW (max_generation, context, fn);
36411 void GCHeap::DiagScanDependentHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
36413 UNREFERENCED_PARAMETER(gen_number);
36414 GCScan::GcScanDependentHandlesForProfilerAndETW (max_generation, context, fn);
36417 // Go through and touch (read) each page straddled by a memory block.
36418 void TouchPages(void * pStart, size_t cb)
36420 const uint32_t pagesize = OS_PAGE_SIZE;
36421 _ASSERTE(0 == (pagesize & (pagesize-1))); // Must be a power of 2.
36424 VOLATILE(char)* pEnd = (VOLATILE(char)*)(cb + (char*)pStart);
36425 VOLATILE(char)* p = (VOLATILE(char)*)(((char*)pStart) - (((size_t)pStart) & (pagesize-1)));
36429 a = VolatileLoad(p);
36430 //printf("Touching page %lxh\n", (uint32_t)p);
36436 #if defined(WRITE_BARRIER_CHECK) && !defined (SERVER_GC)
36437 // This code is designed to catch the failure to update the write barrier
36438 // The way it works is to copy the whole heap right after every GC. The write
36439 // barrier code has been modified so that it updates the shadow as well as the
36440 // real GC heap. Before doing the next GC, we walk the heap, looking for pointers
36441 // that were updated in the real heap, but not the shadow. A mismatch indicates
36442 // an error. The offending code can be found by breaking after the correct GC,
36443 // and then placing a data breakpoint on the Heap location that was updated without
36444 // going through the write barrier.
36446 // Called at process shutdown
36447 void deleteGCShadow()
36449 if (g_GCShadow != 0)
36450 GCToOSInterface::VirtualRelease (g_GCShadow, g_GCShadowEnd - g_GCShadow);
36455 // Called at startup and right after a GC, get a snapshot of the GC Heap
36456 void initGCShadow()
36458 if (!(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK))
36461 size_t len = g_gc_highest_address - g_gc_lowest_address;
36462 if (len > (size_t)(g_GCShadowEnd - g_GCShadow))
36465 g_GCShadowEnd = g_GCShadow = (uint8_t *)GCToOSInterface::VirtualReserve(len, 0, VirtualReserveFlags::None);
36466 if (g_GCShadow == NULL || !GCToOSInterface::VirtualCommit(g_GCShadow, len))
36468 _ASSERTE(!"Not enough memory to run HeapVerify level 2");
36469 // If after the assert we decide to allow the program to continue
36470 // running we need to be in a state that will not trigger any
36471 // additional AVs while we fail to allocate a shadow segment, i.e.
36472 // ensure calls to updateGCShadow() checkGCWriteBarrier() don't AV
36477 g_GCShadowEnd += len;
36480 // save the value of g_gc_lowest_address at this time. If this value changes before
36481 // the next call to checkGCWriteBarrier() it means we extended the heap (with a
36482 // large object segment most probably), and the whole shadow segment is inconsistent.
36483 g_shadow_lowest_address = g_gc_lowest_address;
36485 //****** Copy the whole GC heap ******
36487 // NOTE: This is the one situation where the combination of heap_segment_rw(gen_start_segment())
36488 // can produce a NULL result. This is because the initialization has not completed.
36490 generation* gen = gc_heap::generation_of (max_generation);
36491 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
36493 ptrdiff_t delta = g_GCShadow - g_gc_lowest_address;
36494 BOOL small_object_segments = TRUE;
36499 if (small_object_segments)
36501 small_object_segments = FALSE;
36502 seg = heap_segment_rw (generation_start_segment (gc_heap::generation_of (max_generation+1)));
36508 // Copy the segment
36509 uint8_t* start = heap_segment_mem(seg);
36510 uint8_t* end = heap_segment_allocated (seg);
36511 memcpy(start + delta, start, end - start);
36512 seg = heap_segment_next_rw (seg);
36516 #define INVALIDGCVALUE (void*)((size_t)0xcccccccd)
36518 // test to see if 'ptr' was only updated via the write barrier.
36519 inline void testGCShadow(Object** ptr)
36521 Object** shadow = (Object**) &g_GCShadow[((uint8_t*) ptr - g_gc_lowest_address)];
36522 if (*ptr != 0 && (uint8_t*) shadow < g_GCShadowEnd && *ptr != *shadow)
36525 // If you get this assertion, someone updated a GC pointer in the heap without
36526 // using the write barrier. To find out who, check the value of
36527 // dd_collection_count (dynamic_data_of (0)). Also
36528 // note the value of 'ptr'. Rerun the App that the previous GC just occurred.
36529 // Then put a data breakpoint for the value of 'ptr' Then check every write
36530 // to pointer between the two GCs. The last one is not using the write barrier.
36532 // If the memory of interest does not exist at system startup,
36533 // you need to set the data breakpoint right after the memory gets committed
36534 // Set a breakpoint at the end of grow_heap_segment, and put the value of 'ptr'
36535 // in the memory window. run until the memory gets mapped. Then you can set
36538 // Note a recent change, we've identified race conditions when updating the gc shadow.
36539 // Throughout the runtime, code will update an address in the gc heap, then erect the
36540 // write barrier, which calls updateGCShadow. With an app that pounds one heap location
36541 // from multiple threads, you can hit this assert even though all involved are using the
36542 // write barrier properly. Thusly, we detect the race and set this location to INVALIDGCVALUE.
36543 // TODO: the code in jithelp.asm doesn't call updateGCShadow, and hasn't been
36544 // TODO: fixed to detect the race. We've only seen this race from VolatileWritePtr,
36545 // TODO: so elect not to fix jithelp.asm at this time. It should be done if we start hitting
36546 // TODO: erroneous asserts in here.
36548 if(*shadow!=INVALIDGCVALUE)
36550 #ifdef FEATURE_BASICFREEZE
36551 // Write barriers for stores of references to frozen objects may be optimized away.
36552 if (!gc_heap::frozen_object_p(*ptr))
36553 #endif // FEATURE_BASICFREEZE
36555 _ASSERTE(!"Pointer updated without using write barrier");
36561 printf("saw a INVALIDGCVALUE. (just to let you know)\n");
36567 void testGCShadowHelper (uint8_t* x)
36569 size_t s = size (x);
36570 if (contain_pointers (x))
36572 go_through_object_nostart (method_table(x), x, s, oo,
36573 { testGCShadow((Object**) oo); });
36577 // Walk the whole heap, looking for pointers that were not updated with the write barrier.
36578 void checkGCWriteBarrier()
36580 // g_shadow_lowest_address != g_gc_lowest_address means the GC heap was extended by a segment
36581 // and the GC shadow segment did not track that change!
36582 if (g_GCShadowEnd <= g_GCShadow || g_shadow_lowest_address != g_gc_lowest_address)
36584 // No shadow stack, nothing to check.
36589 generation* gen = gc_heap::generation_of (max_generation);
36590 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
36592 PREFIX_ASSUME(seg != NULL);
36596 uint8_t* x = heap_segment_mem(seg);
36597 while (x < heap_segment_allocated (seg))
36599 size_t s = size (x);
36600 testGCShadowHelper (x);
36603 seg = heap_segment_next_rw (seg);
36608 // go through large object heap
36609 int alignment = get_alignment_constant(FALSE);
36610 generation* gen = gc_heap::generation_of (max_generation+1);
36611 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
36613 PREFIX_ASSUME(seg != NULL);
36617 uint8_t* x = heap_segment_mem(seg);
36618 while (x < heap_segment_allocated (seg))
36620 size_t s = size (x);
36621 testGCShadowHelper (x);
36622 x = x + Align (s, alignment);
36624 seg = heap_segment_next_rw (seg);
36628 #endif //WRITE_BARRIER_CHECK && !SERVER_GC
36630 #endif // !DACCESS_COMPILE
36632 #ifdef FEATURE_BASICFREEZE
36633 void gc_heap::walk_read_only_segment(heap_segment *seg, void *pvContext, object_callback_func pfnMethodTable, object_callback_func pfnObjRef)
36635 #ifdef DACCESS_COMPILE
36636 UNREFERENCED_PARAMETER(seg);
36637 UNREFERENCED_PARAMETER(pvContext);
36638 UNREFERENCED_PARAMETER(pfnMethodTable);
36639 UNREFERENCED_PARAMETER(pfnObjRef);
36641 uint8_t *o = heap_segment_mem(seg);
36643 // small heap alignment constant
36644 int alignment = get_alignment_constant(TRUE);
36646 while (o < heap_segment_allocated(seg))
36648 pfnMethodTable(pvContext, o);
36650 if (contain_pointers (o))
36652 go_through_object_nostart (method_table (o), o, size(o), oo,
36655 pfnObjRef(pvContext, oo);
36660 o += Align(size(o), alignment);
36662 #endif //!DACCESS_COMPILE
36664 #endif // FEATURE_BASICFREEZE
36666 #ifndef DACCESS_COMPILE
36667 HRESULT GCHeap::WaitUntilConcurrentGCCompleteAsync(int millisecondsTimeout)
36669 #ifdef BACKGROUND_GC
36670 if (recursive_gc_sync::background_running_p())
36672 uint32_t dwRet = pGenGCHeap->background_gc_wait(awr_ignored, millisecondsTimeout);
36673 if (dwRet == WAIT_OBJECT_0)
36675 else if (dwRet == WAIT_TIMEOUT)
36676 return HRESULT_FROM_WIN32(ERROR_TIMEOUT);
36678 return E_FAIL; // It is not clear if what the last error would be if the wait failed,
36679 // as there are too many layers in between. The best we can do is to return E_FAIL;
36685 #endif // !DACCESS_COMPILE
36687 void GCHeap::TemporaryEnableConcurrentGC()
36689 #ifdef BACKGROUND_GC
36690 gc_heap::temp_disable_concurrent_p = false;
36691 #endif //BACKGROUND_GC
36694 void GCHeap::TemporaryDisableConcurrentGC()
36696 #ifdef BACKGROUND_GC
36697 gc_heap::temp_disable_concurrent_p = true;
36698 #endif //BACKGROUND_GC
36701 bool GCHeap::IsConcurrentGCEnabled()
36703 #ifdef BACKGROUND_GC
36704 return (gc_heap::gc_can_use_concurrent && !(gc_heap::temp_disable_concurrent_p));
36707 #endif //BACKGROUND_GC
36710 void GCHeap::SetFinalizeRunOnShutdown(bool value)
36712 g_fFinalizerRunOnShutDown = value;
36715 void PopulateDacVars(GcDacVars *gcDacVars)
36717 #ifndef DACCESS_COMPILE
36718 assert(gcDacVars != nullptr);
36720 gcDacVars->major_version_number = 1;
36721 gcDacVars->minor_version_number = 0;
36722 gcDacVars->built_with_svr = &g_built_with_svr_gc;
36723 gcDacVars->build_variant = &g_build_variant;
36724 gcDacVars->gc_structures_invalid_cnt = const_cast<int32_t*>(&GCScan::m_GcStructuresInvalidCnt);
36725 gcDacVars->generation_size = sizeof(generation);
36726 gcDacVars->max_gen = &g_max_generation;
36727 #ifndef MULTIPLE_HEAPS
36728 gcDacVars->mark_array = &gc_heap::mark_array;
36729 gcDacVars->ephemeral_heap_segment = reinterpret_cast<dac_heap_segment**>(&gc_heap::ephemeral_heap_segment);
36730 gcDacVars->current_c_gc_state = const_cast<c_gc_state*>(&gc_heap::current_c_gc_state);
36731 gcDacVars->saved_sweep_ephemeral_seg = reinterpret_cast<dac_heap_segment**>(&gc_heap::saved_sweep_ephemeral_seg);
36732 gcDacVars->saved_sweep_ephemeral_start = &gc_heap::saved_sweep_ephemeral_start;
36733 gcDacVars->background_saved_lowest_address = &gc_heap::background_saved_lowest_address;
36734 gcDacVars->background_saved_highest_address = &gc_heap::background_saved_highest_address;
36735 gcDacVars->alloc_allocated = &gc_heap::alloc_allocated;
36736 gcDacVars->next_sweep_obj = &gc_heap::next_sweep_obj;
36737 gcDacVars->oom_info = &gc_heap::oom_info;
36738 gcDacVars->finalize_queue = reinterpret_cast<dac_finalize_queue**>(&gc_heap::finalize_queue);
36739 gcDacVars->generation_table = reinterpret_cast<dac_generation**>(&gc_heap::generation_table);
36740 #ifdef GC_CONFIG_DRIVEN
36741 gcDacVars->gc_global_mechanisms = reinterpret_cast<size_t**>(&gc_global_mechanisms);
36742 gcDacVars->interesting_data_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_data_per_heap);
36743 gcDacVars->compact_reasons_per_heap = reinterpret_cast<size_t**>(&gc_heap::compact_reasons_per_heap);
36744 gcDacVars->expand_mechanisms_per_heap = reinterpret_cast<size_t**>(&gc_heap::expand_mechanisms_per_heap);
36745 gcDacVars->interesting_mechanism_bits_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_mechanism_bits_per_heap);
36746 #endif // GC_CONFIG_DRIVEN
36747 #ifdef HEAP_ANALYZE
36748 gcDacVars->internal_root_array = &gc_heap::internal_root_array;
36749 gcDacVars->internal_root_array_index = &gc_heap::internal_root_array_index;
36750 gcDacVars->heap_analyze_success = &gc_heap::heap_analyze_success;
36751 #endif // HEAP_ANALYZE
36753 gcDacVars->n_heaps = &gc_heap::n_heaps;
36754 gcDacVars->g_heaps = reinterpret_cast<dac_gc_heap***>(&gc_heap::g_heaps);
36755 #endif // MULTIPLE_HEAPS
36756 #endif // DACCESS_COMPILE