JVM Internals - G1 garbage collector

// memAllocator.cpp
HeapWord* MemAllocator::mem_allocate(Allocation& allocation) const {
  // 1. UseTLAB: bump the current TLAB. Success returns the object.
  //    The object does not fit, or UseTLAB is false: continue.
  if (UseTLAB) {
    HeapWord* mem = mem_allocate_inside_tlab_fast();
    if (mem != nullptr) {
      return mem;
    }
  }

  // 2. Anything after this point may block at a safepoint. Checked in debug builds.
  DEBUG_ONLY(allocation._thread->check_for_valid_safepoint_state());

  // 3. UseTLAB: refill the TLAB and place the object in the new buffer.
  //    A null return means "do not use a TLAB for this object": continue.
  if (UseTLAB) {
    HeapWord* mem = mem_allocate_inside_tlab_slow(allocation);
    if (mem != nullptr) {
      return mem;
    }
  }

  // 4. One object from the shared heap. The thread's TLAB, if any, is left as it is.
  return mem_allocate_outside_tlab(allocation);
}

// memAllocator.cpp
HeapWord* MemAllocator::mem_allocate_inside_tlab_fast() const {
  return _thread->tlab().allocate(_word_size);
}

// threadLocalAllocBuffer.inline.hpp
inline HeapWord* ThreadLocalAllocBuffer::allocate(size_t size) {
  HeapWord* obj = top();
  // 1. end() - top() is large enough: publish the new top and return the old one.
  if (pointer_delta(end(), obj) >= size) {
    set_top(obj + size);
    return obj;
  }
  // 2. The object does not fit. The TLAB is unchanged.
  return nullptr;
}

// memAllocator.cpp
HeapWord* MemAllocator::mem_allocate_inside_tlab_slow(Allocation& allocation) const {
  HeapWord* mem = nullptr;
  ThreadLocalAllocBuffer& tlab = _thread->tlab();

  // 1. JVMTI allocation sampling pulls end() back to the sample point, so the fast path
  //    can fail while the real TLAB still has room. Restore that end and retry once.
  if (JvmtiExport::should_post_sampled_object_alloc()) {
    tlab.set_back_allocation_end();
    mem = tlab.allocate(_word_size);
    if (mem != nullptr) {
      return mem;
    }
  }

  // 2. Leftover space is above the waste limit (initially desired_size / TLABRefillWasteFraction).
  //    Keep this TLAB. record_slow_allocation raises the limit so the same size does not
  //    take this branch on every later call. Caller allocates this object outside the TLAB.
  if (tlab.free() > tlab.refill_waste_limit()) {
    tlab.record_slow_allocation(_word_size);
    return nullptr;
  }

  // 3. The remainder is small enough to discard. Record it and retire the TLAB,
  //    which fills the gap with a dummy object.
  tlab.record_refill_waste();
  _thread->retire_tlab();

  // 4. compute_size returns 0 when the object plus the alignment reserve is larger than
  //    the maximum TLAB. A humongous object, larger than half a G1 region, takes this branch.
  size_t new_tlab_size = tlab.compute_size(_word_size);
  if (new_tlab_size == 0) {
    return nullptr;
  }

  // 5. Ask the heap for a new TLAB of any size between min_tlab_size and new_tlab_size.
  //    Failure does not collect; the caller retries through mem_allocate_outside_tlab,
  //    which is allowed to stop for a GC.
  size_t min_tlab_size = ThreadLocalAllocBuffer::compute_min_size(_word_size);
  mem = Universe::heap()->allocate_new_tlab(
      min_tlab_size, new_tlab_size, &allocation._allocated_tlab_size);
  if (mem == nullptr) {
    return nullptr;
  }

  // 6. ZeroTLAB clears the new buffer. ZapTLAB fills it with badHeapWordVal except the
  //    header, so a concurrent collector does not parse the unallocated tail as an object.
  //    Neither flag: the buffer is left uninitialized.
  if (ZeroTLAB) {
    Copy::zero_to_words(mem, allocation._allocated_tlab_size);
  } else if (ZapTLAB) {
    size_t hdr_size = oopDesc::header_size();
    Copy::fill_to_words(mem + hdr_size, allocation._allocated_tlab_size - hdr_size, badHeapWordVal);
  }

  // 7. Install the buffer as this thread's TLAB and bump its top past this object.
  _thread->fill_tlab(mem, _word_size, allocation._allocated_tlab_size);
  return mem;
}

// g1CollectedHeap.cpp
HeapWord* G1CollectedHeap::allocate_new_tlab(size_t min_size,
                                             size_t requested_size,
                                             size_t* actual_size) {
  // A humongous request never reaches here; mem_allocate_inside_tlab_slow returned null at step 4.
  assert(!is_humongous(requested_size), "we do not allow humongous TLABs");
  // No collection. Null means the Eden region could not supply the TLAB.
  return attempt_allocation(min_size, requested_size, actual_size, false /* allow_gc */);
}

// memAllocator.cpp
HeapWord* MemAllocator::mem_allocate_outside_tlab(Allocation& allocation) const {
  allocation._allocated_outside_tlab = true;
  // 1. One object from the heap. G1 may stop for a GC on this call.
  HeapWord* mem = Universe::heap()->mem_allocate(_word_size);
  // 2. Still no space: the caller reports allocation failure.
  if (mem == nullptr) {
    return mem;
  }

  // 3. Count the bytes on the thread and on the heap sampler. The TLAB statistics are unchanged.
  size_t size_in_bytes = _word_size * HeapWordSize;
  _thread->incr_allocated_bytes(size_in_bytes);
  _thread->heap_sampler().inc_outside_tlab_bytes(size_in_bytes);
  return mem;
}

// g1CollectedHeap.cpp
HeapWord* G1CollectedHeap::mem_allocate(size_t word_size) {
  // 1. Larger than half a region: one or more free regions, not the current Eden region.
  if (is_humongous(word_size)) {
    return attempt_allocation_humongous(word_size);
  }
  // 2. Otherwise a single block, min size equal to the object. allow_gc is true,
  //    so a failed attempt can run an evacuation pause and retry.
  size_t dummy = 0;
  return attempt_allocation(word_size, word_size, &dummy, true /* allow_gc */);
}

// g1CollectedHeap.hpp
static bool G1CollectedHeap::is_humongous(size_t word_size) {
  return word_size > _humongous_object_threshold_in_words;
}

// The caller blocks in wait_until_executed until the VM thread finishes the operation.
void VMThread::execute(VM_Operation* op) {
  Thread* t = Thread::current();
  if (t->is_VM_thread()) {
    op->set_calling_thread(t);
    ((VMThread*)t)->inner_execute(op);
    return;
  }
  if (!op->doit_prologue()) {
    return;
  }
  op->set_calling_thread(t);
  wait_until_executed(op);
  op->doit_epilogue();
}

// VM thread only: open a safepoint, run the operation, then close it.
void VMThread::inner_execute(VM_Operation* op) {
  assert(Thread::current()->is_VM_thread(), "Must be the VM thread");
  _cur_vm_operation = op;

  bool end_safepoint = false;
  if (_cur_vm_operation->evaluate_at_safepoint() &&
      !SafepointSynchronize::is_at_safepoint()) {
    SafepointSynchronize::begin();
    end_safepoint = true;
  }

  evaluate_operation(_cur_vm_operation);

  if (end_safepoint) {
    SafepointSynchronize::end();
  }
  _cur_vm_operation = nullptr;
}

// VM thread: yield suspendible GC threads, arm every poll, wait until no Java thread is running.
void SafepointSynchronize::begin() {
  assert(Thread::current()->is_VM_thread(), "Only VM thread may execute a safepoint");

  Universe::heap()->safepoint_synchronize_begin();
  Threads_lock->lock();

  int nof_threads = Threads::number_of_threads();
  _waiting_to_block = nof_threads;

  jlong safepoint_limit_time = 0;
  int initial_running = 0;
  arm_safepoint();
  synchronize_threads(safepoint_limit_time, nof_threads, &initial_running);
  assert(_waiting_to_block == 0, "No thread should be running");

  _state = _synchronized;
}

// G1: the marker and refinement threads wait out this safepoint.
void G1CollectedHeap::safepoint_synchronize_begin() {
  SuspendibleThreadSet::synchronize();
}

// Publish this safepoint and arm the local poll of every Java thread.
void SafepointSynchronize::arm_safepoint() {
  _wait_barrier->arm(static_cast<int>(_safepoint_counter + 1));
  AtomicAccess::release_store(&_safepoint_counter, _safepoint_counter + 1);

  OrderAccess::storestore();
  _state = _synchronizing;

  OrderAccess::storestore();
  for (JavaThreadIteratorWithHandle jtiwh; JavaThread* cur = jtiwh.next(); ) {
    SafepointMechanism::arm_local_poll(cur);
  }
  OrderAccess::fence();
}

// Armed polling word (low bit) and armed polling page for one thread.
void SafepointMechanism::arm_local_poll(JavaThread* thread) {
  thread->poll_data()->set_polling_word(_poll_word_armed_value);
  thread->poll_data()->set_polling_page(_poll_page_armed_value);
}

// User-thread entry. A return bytecode on this thread tests the poll.
// The armed poll calls InterpreterRuntime::at_safepoint before the frame is removed.
void TemplateTable::_return(TosState state) {
  transition(state, state);

  Label no_safepoint;
  __ testb(Address(r15_thread, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
  __ jcc(Assembler::zero, no_safepoint);
  __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::at_safepoint));
  __ bind(no_safepoint);
}

// JRT_ENTRY. The destructor runs on this same thread as the call returns to Java.
class ThreadInVMfromJava : public ThreadStateTransition {
  ThreadInVMfromJava(JavaThread* thread, bool check_asyncs = true) : ThreadStateTransition(thread), _check_asyncs(check_asyncs) {
    transition_from_java(thread, _thread_in_vm);
  }
  ~ThreadInVMfromJava() {
    transition_from_vm(_thread, _thread_in_Java, _check_asyncs);
  }
};

// Interpreter back-edge or return. The x86 poll tests this thread's polling word.
// A set poll bit jumps to the slow path, which calls InterpreterRuntime::at_safepoint.
void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool in_nmethod) {
  if (at_return) {
    cmpptr(in_nmethod ? rsp : rbp, Address(r15_thread, JavaThread::polling_word_offset()));
    jcc(Assembler::above, slow_path);
    return;
  }
  testb(Address(r15_thread, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
  jcc(Assembler::notZero, slow_path);
}

// JRT_ENTRY wraps the call in ThreadInVMfromJava. Its destructor returns to Java.
JRT_ENTRY(void, InterpreterRuntime::at_safepoint(JavaThread* current))
JRT_END

// ~ThreadInVMfromJava calls this. This is the user thread's safepoint check.
static inline void transition_from_vm(JavaThread* thread, JavaThreadState to, bool check_asyncs = true) {
  assert(thread->thread_state() == _thread_in_vm, "coming from wrong thread state");
  if (to == _thread_in_Java) {
    SafepointMechanism::process_if_requested_with_exit_check(thread, check_asyncs);
    thread->set_thread_state(to);
  }
}

// Returning from native code takes the same check on this thread.
static inline void transition_from_native(JavaThread* thread, JavaThreadState to, bool check_asyncs = true) {
  assert(thread->thread_state() == _thread_in_native, "coming from wrong thread state");
  thread->set_thread_state_fence(_thread_in_vm);
  SafepointMechanism::process_if_requested_with_exit_check(thread, to != _thread_in_Java ? false : check_asyncs);
  thread->set_thread_state(to);
}

void JavaThread::check_special_condition_for_native_trans(JavaThread* thread) {
  assert(thread->thread_state() == _thread_in_native_trans, "wrong state");
  thread->set_thread_state(_thread_in_vm);
  SafepointMechanism::process_if_requested_with_exit_check(thread, true /* check asyncs */);
}

// Armed local poll: this is the call that reaches SafepointSynchronize::block.
void SafepointMechanism::process_if_requested(JavaThread* thread, bool allow_suspend, bool check_async_exception) {
  thread->check_possible_safepoint();
  if (local_poll_armed(thread)) {
    process(thread, allow_suspend, check_async_exception);
  }
}

// User thread: a global safepoint is open, so block until it ends.
void SafepointMechanism::process(JavaThread* thread, bool allow_suspend, bool check_async_exception) {
  bool need_rechecking;
  do {
    if (global_poll()) {
      OrderAccess::loadload();
      SafepointSynchronize::block(thread);
    }
    StackWatermarkSet::on_safepoint(thread);
    need_rechecking = thread->handshake_state()->has_operation() &&
                      thread->handshake_state()->process_by_self(allow_suspend, check_async_exception);
  } while (need_rechecking);
  update_poll_values(thread);
}

// Compiled poll fault. Enter the VM, handle the poll, return to Java.
void SafepointSynchronize::handle_polling_page_exception(JavaThread* thread) {
  assert(thread->thread_state() == _thread_in_Java, "should come from Java code");
  thread->set_thread_state(_thread_in_vm);
  thread->safepoint_state()->handle_polling_page_exception();
  thread->set_thread_state(_thread_in_Java);
}

// Return poll and loop-backedge poll both reach process, which calls block.
void ThreadSafepointState::handle_polling_page_exception() {
  JavaThread* self = thread();
  address real_return_addr = self->saved_exception_pc();
  nmethod* nm = CodeCache::find_blob(real_return_addr)->as_nmethod();

  if (nm->is_at_poll_return(real_return_addr)) {
    SafepointMechanism::process_if_requested_with_exit_check(self, true /* check asyncs */);
  } else {
    set_at_poll_safepoint(true);
    SafepointMechanism::process_if_requested_with_exit_check(self, false /* check asyncs */);
    set_at_poll_safepoint(false);
  }
}

// The stop: stack is walkable, state is _thread_blocked, wait on the barrier.
void SafepointSynchronize::block(JavaThread* thread) {
  JavaThreadState state = thread->thread_state();
  thread->frame_anchor()->make_walkable();

  uint64_t safepoint_id = SafepointSynchronize::safepoint_counter();
  thread->safepoint_state()->set_safepoint_id(safepoint_id);
  OrderAccess::storestore();
  thread->set_thread_state_fence(_thread_blocked);

  _wait_barrier->wait(static_cast<int>(safepoint_id));

  OrderAccess::loadstore();
  thread->set_thread_state(state);
  thread->safepoint_state()->reset_safepoint_id();
}

// VM thread spins until every Java thread is safepoint-safe.
int SafepointSynchronize::synchronize_threads(jlong safepoint_limit_time, int nof_threads, int* initial_running) {
  int still_running = nof_threads;
  ThreadSafepointState* tss_head = nullptr;
  ThreadSafepointState** p_prev = &tss_head;
  for (JavaThreadIteratorWithHandle jtiwh; JavaThread* cur = jtiwh.next(); ) {
    ThreadSafepointState* cur_tss = cur->safepoint_state();
    if (thread_not_running(cur_tss)) {
      --still_running;
    } else {
      *p_prev = cur_tss;
      p_prev = cur_tss->next_ptr();
    }
  }
  *p_prev = nullptr;
  *initial_running = still_running;
  if (still_running <= 0) {
    return 1;
  }

  int iterations = 1;
  int64_t start_time = os::javaTimeNanos();
  do {
    p_prev = &tss_head;
    ThreadSafepointState* cur_tss = tss_head;
    while (cur_tss != nullptr) {
      if (thread_not_running(cur_tss)) {
        --still_running;
        *p_prev = nullptr;
        ThreadSafepointState* tmp = cur_tss;
        cur_tss = cur_tss->get_next();
        tmp->set_next(nullptr);
      } else {
        *p_prev = cur_tss;
        p_prev = cur_tss->next_ptr();
        cur_tss = cur_tss->get_next();
      }
    }
    if (still_running > 0) {
      back_off(start_time);
    }
    iterations++;
  } while (still_running > 0);
  return iterations;
}

// True once this thread has left the running state, or was already safe.
bool SafepointSynchronize::thread_not_running(ThreadSafepointState* cur_state) {
  if (!cur_state->is_running()) {
    return true;
  }
  cur_state->examine_state_of_thread(SafepointSynchronize::safepoint_counter());
  return !cur_state->is_running();
}

// A stable safepoint-safe state (blocked, or in native) counts the thread stopped.
void ThreadSafepointState::examine_state_of_thread(uint64_t safepoint_count) {
  JavaThreadState stable_state;
  if (!SafepointSynchronize::try_stable_load_state(&stable_state, _thread, safepoint_count)) {
    return;
  }
  if (safepoint_safe_with(_thread, stable_state)) {
    account_safe_thread();
  }
}

// One fewer thread for begin to wait on.
void ThreadSafepointState::account_safe_thread() {
  SafepointSynchronize::decrement_waiting_to_block();
  _safepoint_safe = true;
}

// VM thread: disarm polls and let yielding GC threads resume.
void SafepointSynchronize::end() {
  assert(Thread::current()->is_VM_thread(), "Only VM thread can execute a safepoint");
  disarm_safepoint();
  Universe::heap()->safepoint_synchronize_end();
}

// Clear the global state, mark each thread running again, wake the barrier.
void SafepointSynchronize::disarm_safepoint() {
  _state = _not_synchronized;
  AtomicAccess::release_store(&_safepoint_counter, _safepoint_counter + 1);

  for (JavaThreadIteratorWithHandle jtiwh; JavaThread* current = jtiwh.next(); ) {
    current->safepoint_state()->restart();
  }
  Threads_lock->unlock();
  _wait_barrier->disarm();
}

// G1: the marker and refinement threads may run again.
void G1CollectedHeap::safepoint_synchronize_end() {
  SuspendibleThreadSet::desynchronize();
}

// g1CollectedHeap.cpp — the mutator cannot refill Eden
HeapWord* G1CollectedHeap::attempt_allocation_slow(uint node_index, size_t word_size, bool allow_gc) {
  uint gc_count_before;
  {
    MutexLocker x(Heap_lock);
    // Retry under the lock in case another thread changed the region while we waited.
    result = _allocator->attempt_allocation_locked(node_index, word_size);
    if (result != nullptr) {
      return result;
    }
    gc_count_before = total_collections();   // remember the GC count for the pause
  }

  bool succeeded;
  result = do_collection_pause(word_size, gc_count_before, &succeeded,
                               GCCause::_g1_inc_collection_pause);
  if (succeeded) {
    return result;   // the pause freed space; the mutator allocates from it
  }
}

// g1CollectedHeap.cpp — on the allocating mutator
HeapWord* G1CollectedHeap::do_collection_pause(size_t word_size,
                                               uint gc_count_before,
                                               bool* succeeded,
                                               GCCause::Cause gc_cause) {
  VM_G1CollectForAllocation op(word_size, gc_count_before, gc_cause);
  VMThread::execute(&op);   // the allocating mutator blocks here
}

// g1VMOperations.cpp — on the VM thread
void VM_G1CollectForAllocation::doit() {
  G1CollectedHeap* g1h = G1CollectedHeap::heap();
  GCCauseSetter x(g1h, _gc_cause);
  // Try a partial collection of some kind.
  g1h->do_collection_pause_at_safepoint(_word_size);

  if (_word_size > 0) {
    // An allocation had been requested. Do it, eventually trying a stronger kind of GC.
    _result = g1h->satisfy_failed_allocation(_word_size);
  }
}

// g1CollectedHeap.cpp — on the VM thread
void G1CollectedHeap::do_collection_pause_at_safepoint(size_t allocation_word_size) {
  policy()->decide_on_concurrent_start_pause();
  bool should_start_concurrent_mark_operation = collector_state()->is_in_concurrent_start_gc();

  G1YoungCollector collector(gc_cause(), allocation_word_size);
  collector.collect();
  _collector_state = collector.next_state();   // the state the next pause reads

  if (should_start_concurrent_mark_operation) {
    // This pause is the Concurrent Start: start the marking cycle.
    start_concurrent_cycle(collector.concurrent_operation_is_full_mark());
  }
}

// workerThread.cpp
void WorkerThreads::initialize_workers() {
  const uint initial_active_workers = UseDynamicNumberOfGCThreads ? 1 : _max_workers;
  if (set_active_workers(initial_active_workers) != initial_active_workers) {
    vm_exit_during_initialization();
  }
}

uint WorkerThreads::set_active_workers(uint num_workers) {
  // Create the missing workers on demand.
  while (_created_workers < num_workers) {
    WorkerThread* const worker = create_worker(_created_workers);
    ...
    _workers[_created_workers] = worker;
    _created_workers++;
  }
  _active_workers = MIN2(_created_workers, num_workers);
}

// workerThread.cpp — called on the driver; returns when the task is done
void WorkerThreads::run_task(WorkerTask* task) {
  _dispatcher.coordinator_distribute_task(task, _active_workers);
}

// workerThread.cpp — driver side
void WorkerTaskDispatcher::coordinator_distribute_task(WorkerTask* task, uint num_workers) {
  // 1. Publish the task and how many workers must report done.
  _task = task;
  _not_finished.store_relaxed(num_workers);
  // 2. Wake that many workers.
  _start_semaphore.signal(num_workers);
  // 3. Block until the last worker signals back. The phase ends here.
  _end_semaphore.wait();
  ...
}

// workerThread.cpp — worker side, called in a loop
void WorkerTaskDispatcher::worker_run_task() {
  // 1. Park until a task is dispatched. This is what idle workers do.
  _start_semaphore.wait();
  // 2. Claim the next dense id, in wake-up order: 0, 1, ..., n-1.
  const uint worker_id = _started.fetch_then_add(1u);
  WorkerThread::set_worker_id(worker_id);
  // 3. Enter the task's GC id and run this worker's slice.
  GCIdMark gc_id_mark(_task->gc_id());
  _task->work(worker_id);
  // 4. Report done; the last worker out wakes the driver.
  const uint not_finished = _not_finished.sub_then_fetch(1u);
  if (not_finished == 0) {
    _end_semaphore.signal();
  }
}

// workerThread.cpp — the entire life of a worker
void WorkerThread::run() {
  while (true) {
    _dispatcher->worker_run_task();
  }
}

// g1CollectedHeap.cpp
jint G1CollectedHeap::initialize() {
  _workers = new WorkerThreads("GC Thread", ParallelGCThreads);
  _workers->initialize_workers();
}

// The allocating thread blocks until the VM thread has finished the operation.
void VMThread::execute(VM_Operation* op) {
  Thread* t = Thread::current();
  if (t->is_VM_thread()) {
    op->set_calling_thread(t);
    ((VMThread*)t)->inner_execute(op);
    return;
  }
  if (!op->doit_prologue()) {
    return;
  }
  op->set_calling_thread(t);
  wait_until_executed(op);
  op->doit_epilogue();
}

// VM thread: stop every Java thread, run doit, then release the safepoint.
void VMThread::inner_execute(VM_Operation* op) {
  assert(Thread::current()->is_VM_thread(), "Must be the VM thread");
  _cur_vm_operation = op;

  bool end_safepoint = false;
  if (_cur_vm_operation->evaluate_at_safepoint() &&
      !SafepointSynchronize::is_at_safepoint()) {
    SafepointSynchronize::begin();
    end_safepoint = true;
  }
  evaluate_operation(_cur_vm_operation);
  if (end_safepoint) {
    SafepointSynchronize::end();
  }
  _cur_vm_operation = nullptr;
}

// One young or mixed pause, then the allocation that requested it retries.
void VM_G1CollectForAllocation::doit() {
  G1CollectedHeap* g1h = G1CollectedHeap::heap();
  GCCauseSetter x(g1h, _gc_cause);
  g1h->do_collection_pause_at_safepoint(_word_size);
  if (_word_size > 0) {
    _result = g1h->satisfy_failed_allocation(_word_size);
  }
}

// Build the young collector and run it. Start marking only if this pause is a Concurrent Start.
void G1CollectedHeap::do_collection_pause_at_safepoint(size_t allocation_word_size) {
  policy()->decide_on_concurrent_start_pause();
  bool should_start_concurrent_mark_operation = collector_state()->is_in_concurrent_start_gc();

  G1YoungCollector collector(gc_cause(), allocation_word_size);
  collector.collect();
  _collector_state = collector.next_state();

  if (should_start_concurrent_mark_operation) {
    start_concurrent_cycle(collector.concurrent_operation_is_full_mark());
  }
}

// Serial setup, then one run_task per parallel phase. The VM thread waits inside each run_task.
void G1YoungCollector::collect() {
  set_young_collection_default_active_worker_threads();
  complete_root_region_scan();
  policy()->record_young_collection_start();

  pre_evacuate_collection_set(evacuation_info);
  G1ParScanThreadStateSet per_thread_states(_g1h, workers()->active_workers(), collection_set(), &_evac_failure_regions);

  bool may_do_optional_evacuation = collection_set()->num_optional_regions() != 0;
  evacuate_initial_collection_set(&per_thread_states, may_do_optional_evacuation);
  if (may_do_optional_evacuation) {
    evacuate_optional_collection_set(&per_thread_states);
  }
  post_evacuate_collection_set(evacuation_info, &per_thread_states);

  _next_state = policy()->record_young_collection_end(_concurrent_operation_is_full_mark, evacuation_alloc_failed(), _allocation_word_size);
}

// Driver side of every parallel phase: return only after the workers finish.
Tickspan G1YoungCollector::run_task_timed(WorkerTask* task) {
  Ticks start = Ticks::now();
  workers()->run_task(task);
  return Ticks::now() - start;
}

// A batched task asks for its own worker count, then the same run_task.
void G1CollectedHeap::run_batch_task(G1BatchedTask* cl) {
  uint num_workers = MAX2(1u, MIN2(cl->num_workers_estimate(), workers()->active_workers()));
  cl->set_max_workers(num_workers);
  workers()->run_task(cl, num_workers);
}

// Before the copy: flush thread-local buffers, choose the collection set, then one prepare task.
void G1YoungCollector::pre_evacuate_collection_set(G1EvacInfo* evacuation_info) {
  G1PreEvacuateCollectionSetBatchTask cl;
  G1CollectedHeap::heap()->run_batch_task(&cl);

  calculate_collection_set(evacuation_info, policy()->max_pause_time_ms());
  if (collector_state()->is_in_concurrent_start_gc()) {
    concurrent_mark()->pre_concurrent_start(_gc_cause);
  }
  ref_processor_stw()->start_discovery(false /* always_clear */);
  allocator()->init_gc_alloc_regions(evacuation_info);
  rem_set()->prepare_for_scan_heap_roots();
  _g1h->collection_set()->prepare_for_scan();

  G1PrepareEvacuationTask g1_prep_task(_g1h);
  run_task_timed(&g1_prep_task);
}

// The copy: merge remembered-set roots, then one evacuate task on the pool.
void G1YoungCollector::evacuate_initial_collection_set(G1ParScanThreadStateSet* per_thread_states,
                                                      bool has_optional_evacuation_work) {
  rem_set()->merge_heap_roots(true /* initial_evacuation */);
  const uint num_workers = workers()->active_workers();
  G1RootProcessor root_processor(_g1h, num_workers > 1 /* is_parallel */);
  G1EvacuateRegionsTask g1_par_task(_g1h, per_thread_states, task_queues(), &root_processor,
                                    num_workers, has_optional_evacuation_work);
  run_task_timed(&g1_par_task);
  rem_set()->complete_evac_phase(has_optional_evacuation_work);
}

// Optional old regions, only while the pause budget remains.
void G1YoungCollector::evacuate_next_optional_regions(G1ParScanThreadStateSet* per_thread_states) {
  G1EvacuateOptionalRegionsTask task(per_thread_states, task_queues(), workers()->active_workers());
  run_task_timed(&task);
}

// After the copy: references, weak oops, then two cleanup tasks on the pool.
void G1YoungCollector::post_evacuate_collection_set(G1EvacInfo* evacuation_info,
                                                    G1ParScanThreadStateSet* per_thread_states) {
  process_discovered_references(per_thread_states);
  G1STWIsAliveClosure is_alive(_g1h);
  G1KeepAliveClosure keep_alive(_g1h);
  WeakProcessor::weak_oops_do(workers(), &is_alive, &keep_alive, phase_times()->weak_phase_times());
  allocator()->release_gc_alloc_regions(evacuation_info);
  post_evacuate_cleanup_1(per_thread_states);
  post_evacuate_cleanup_2(per_thread_states, evacuation_info);
}

void G1YoungCollector::post_evacuate_cleanup_1(G1ParScanThreadStateSet* per_thread_states) {
  G1PostEvacuateCollectionSetCleanupTask1 cl(per_thread_states, &_evac_failure_regions);
  _g1h->run_batch_task(&cl);
}

void G1YoungCollector::post_evacuate_cleanup_2(G1ParScanThreadStateSet* per_thread_states,
                                               G1EvacInfo* evacuation_info) {
  G1PostEvacuateCollectionSetCleanupTask2 cl(per_thread_states, evacuation_info, &_evac_failure_regions);
  _g1h->run_batch_task(&cl);
}

// One worker. Private scan state, then roots, then the copy loop.
void G1EvacuateRegionsBaseTask::work(uint worker_id) {
  start_work(worker_id);
  {
    ResourceMark rm;
    G1ParScanThreadState* pss = _per_thread_states->state_for_worker(worker_id);
    pss->set_ref_discoverer(_g1h->ref_processor_stw());
    if (_pinned_regions_recorded.compare_set(false, true)) {
      record_pinned_regions(pss, worker_id);
    }
    scan_roots(pss, worker_id);
    evacuate_live_objects(pss, worker_id);
  }
  end_work(worker_id);
}

// Roots of the initial collection set: thread stacks, heap roots, code roots.
void G1EvacuateRegionsTask::scan_roots(G1ParScanThreadState* pss, uint worker_id) {
  _root_processor->evacuate_roots(pss, worker_id);
  _g1h->rem_set()->scan_heap_roots(pss, worker_id, G1GCPhaseTimes::ScanHR, G1GCPhaseTimes::ObjCopy, _has_optional_evacuation_work);
  _g1h->rem_set()->scan_collection_set_code_roots(pss, worker_id, G1GCPhaseTimes::CodeRoots, G1GCPhaseTimes::ObjCopy);
}

// Drain this worker's queue, steal from the others, stop when every queue is empty.
void G1ParEvacuateFollowersClosure::do_void() {
  G1ParScanThreadState* const pss = par_scan_state();
  pss->trim_queue();
  do {
    pss->steal_and_trim_queue(queues());
  } while (!offer_termination());
}

bool G1ParEvacuateFollowersClosure::offer_termination() {
  return (terminator() == nullptr) ? true : terminator()->offer_termination();
}

// g1CollectedHeap.cpp — the VM thread schedules a cycle
void G1CollectedHeap::start_concurrent_cycle(bool concurrent_operation_is_full_mark) {
  MutexLocker x(G1CGC_lock, Mutex::_no_safepoint_check_flag);
  if (concurrent_operation_is_full_mark) {
    _cm->start_full_concurrent_cycle();   // prepare the cycle (§4.3)
  } else {
    _cm->start_undo_concurrent_cycle();
  }
  G1CGC_lock->notify();   // wake the marker thread
}

// g1ConcurrentMark.cpp
void G1ConcurrentMark::fully_initialize() {
  _cm_thread = new G1ConcurrentMarkThread(this);
  _max_concurrent_workers = ConcGCThreads;
  _concurrent_workers = new WorkerThreads("G1 Conc", _max_concurrent_workers);
  _concurrent_workers->initialize_workers();
}

// g1ConcurrentMarkThread.cpp
void G1ConcurrentMarkThread::run_service() {
  while (wait_for_next_cycle()) {
    ...
    if (state() == FullCycleMarking) {
      concurrent_mark_cycle_do();
    } else {
      concurrent_undo_cycle_do();
    }
    ...
  }
}

// g1ConcurrentMark.cpp — called by the marker thread
void G1ConcurrentMark::mark_from_roots() {
  G1CMConcurrentMarkingTask marking_task(this);
  _concurrent_workers->run_task(&marking_task);   // marker thread blocks here
}

// g1ConcurrentMarkThread.cpp
bool G1ConcurrentMarkThread::subphase_remark() {
  VM_G1PauseRemark op;
  VMThread::execute(&op);   // marker thread blocks; VM thread runs the pause
}

// g1ConcurrentRefineThread.cpp — the control thread's idle wait
bool G1ConcurrentRefineThread::wait_for_work() {
  MonitorLocker ml(notifier(), Mutex::_no_safepoint_check_flag);
  if (!requested_active() && !should_terminate()) {
    ml.wait(cr()->adjust_threads_wait_ms());   // timed: wake up and re-check
  }
  cr()->record_thread_adjustment_needed();
  return !should_terminate();
}

// g1ConcurrentRefineThread.cpp — explicit wake-up, called after a collection
void G1ConcurrentRefineThread::activate() {
  MonitorLocker ml(&_notifier, Mutex::_no_safepoint_check_flag);
  if (!_requested_active || should_terminate()) {
    _requested_active = true;
    ml.notify();
  }
}

// g1ConcurrentRefine.cpp
jint G1ConcurrentRefineThreadControl::initialize(G1ConcurrentRefine* cr) {
  _control_thread = create_refinement_thread();
  _workers = new WorkerThreads("G1 Refinement Workers", max_num_threads());
  _workers->initialize_workers();
}

// g1ConcurrentRefineThread.cpp
void G1ConcurrentRefineThread::run_service() {
  while (wait_for_work()) {                       // timed wait while idle
    ...
    if (cr()->adjust_num_threads_periodically()) {
      do_refinement();
    } ...
  }
}

// g1ConcurrentRefine.cpp
void G1ConcurrentRefine::run_with_refinement_workers(WorkerTask* task) {
  _thread_control.run_task(task, num_threads_wanted());
}

// g1ConcurrentRefineSweepTask.cpp — worker side
void G1ConcurrentRefineSweepTask::work(uint worker_id) {
  G1RefineRegionClosure sweep_cl(worker_id, _scan_state);
  _scan_state->heap_region_iterate_from_worker_offset(&sweep_cl, worker_id, _max_workers);
  ...
}

// g1ParScanThreadState.cpp
G1HeapRegionAttr G1ParScanThreadState::next_region_attr(G1HeapRegionAttr const region_attr, markWord const m, uint& age) {
  if (region_attr.is_young()) {
    age = !m.has_displaced_mark_helper() ? m.age()
                                         : m.displaced_mark_helper().age();
    // Below the threshold: stay young, so the copy lands in Survivor.
    if (age < _tenuring_threshold) {
      return region_attr;
    }
  }
  // Young and old enough: promotion. Already old: old-to-old copy.
  return G1HeapRegionAttr::Old;
}

// g1ParScanThreadState.cpp
oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const region_attr,
                                                    oop const old,
                                                    markWord const old_mark) {
  uint age = 0;
  G1HeapRegionAttr dest_attr = next_region_attr(region_attr, old_mark, age);
  HeapWord* obj_ptr = _plab_allocator->plab_allocate(dest_attr, word_sz, node_index);
  // PLAB full: refill, or allocate this one object directly. Both can fail.
  if (obj_ptr == nullptr) {
    obj_ptr = allocate_copy_slow(&dest_attr, klass, word_sz, age, node_index);
  }
  Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(old), obj_ptr, word_sz);
  const oop obj = cast_to_oop(obj_ptr);
  const oop forward_ptr = old->forward_to_atomic(obj, old_mark, memory_order_relaxed);
  // This worker installed the forwarding pointer. Another worker already did: undo and return theirs.
  if (forward_ptr == nullptr) {
    if (dest_attr.is_young()) {
      if (age < markWord::max_age) {
        age++;
        obj->incr_age();
      }
      _age_table.add(age, word_sz);
    } else {
      update_bot_after_copying(obj, word_sz);
    }
  }
}

// g1ParScanThreadState.cpp
HeapWord* G1ParScanThreadState::allocate_in_next_plab(G1HeapRegionAttr* dest,
                                                      size_t word_sz,
                                                      bool previous_plab_refill_failed,
                                                      uint node_index) {
  if (dest->is_young()) {
    HeapWord* const obj_ptr = _plab_allocator->allocate(G1HeapRegionAttr::Old, word_sz,
                                                        &plab_refill_in_old_failed, node_index);
    // A failed Survivor refill means Survivor has no room. Promote from here on.
    if (previous_plab_refill_failed) {
      _tenuring_threshold = 0;
    }
    if (obj_ptr != nullptr) {
      dest->set_old();
    }
    return obj_ptr;
  }
}

// g1ParScanThreadState.cpp — each worker, at the end of the pause
size_t G1ParScanThreadState::flush_stats(size_t* surviving_young_words, uint num_workers) {
  flush_numa_stats();
  _plab_allocator->flush_and_retire_stats(num_workers);
  // Merge this worker's AgeTable into the global one.
  _g1h->policy()->record_age_table(&_age_table);
  ...
}

// g1Policy.cpp — called from record_young_collection_start
void G1Policy::update_survivors_policy() {
  double max_survivor_regions_d = (double)young_list_target_length() / (double)SurvivorRatio;
  uint const desired_max_survivor_regions = ceil(max_survivor_regions_d);
  size_t const survivor_size = desired_survivor_size(desired_max_survivor_regions);
  _tenuring_threshold = _survivors_age_table.compute_tenuring_threshold(survivor_size);
}

// ageTable.cpp — AlwaysTenure and NeverTenure return
// MaxTenuringThreshold immediately. Otherwise sizes[] and the desired size are in words.
uint AgeTable::compute_tenuring_threshold(size_t desired_survivor_size) {
  size_t total = 0;
  uint age = 1;
  while (age < table_size) {
    total += sizes[age];
    // Including this age overflowed the desired Survivor space: this age promotes.
    if (total > desired_survivor_size) break;
    age++;
  }
  return age < MaxTenuringThreshold ? age : MaxTenuringThreshold;
}

// g1ConcurrentMark.cpp — before evacuation, at the same safepoint
void G1ConcurrentMark::pre_concurrent_start(GCCause::Cause cause) {
  assert_at_safepoint_on_vm_thread();

  G1CollectedHeap::start_codecache_marking_cycle_if_inactive(true /* concurrent_mark_start */);
  ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_strong);

  // serial: reset the marking state left by the previous cycle
  // parallel: per old region, take the top-at-mark-start (TAMS) boundary
  G1PreConcurrentStartTask cl(cause, this);
  G1CollectedHeap::heap()->run_batch_task(&cl);
}

// g1RootClosures.cpp — closure set for the pause's root scan
G1EvacuationRootClosures* G1EvacuationRootClosures::create_root_closures(G1CollectedHeap* g1h,
                                                                         G1ParScanThreadState* pss,
                                                                         bool process_only_dirty_klasses) {
  if (g1h->collector_state()->is_in_concurrent_start_gc()) {
    ...
    res = new G1ConcurrentStartMarkClosures<...>(g1h, pss);   // should_mark == true
  } else {
    res = new G1EvacuationClosures(g1h, pss, process_only_dirty_klasses);
  }
}

// g1OopClosures.inline.hpp — one root during the Concurrent Start pause
void G1ParCopyClosure<barrier, should_mark>::do_oop_work(T* p) {
  const G1HeapRegionAttr state = _g1h->region_attr(obj);
  if (state.is_in_cset()) {
    oop forwardee;
    markWord m = obj->mark();
    if (m.is_forwarded()) {
      forwardee = obj->forwardee(m);
    } else {
      forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m);
    }
    RawAccess<IS_NOT_NULL>::oop_store(p, forwardee);   // into young: evacuate
  } else {
    ...
    if (should_mark) {
      mark_object(obj);   // into old: _cm->mark_in_bitmap(_worker_id, obj)
    }
  }
}

// g1ConcurrentMark.cpp
void G1ConcurrentMark::start_full_concurrent_cycle() {
  // 1. VM thread, still in the safepoint: begin weak-reference discovery for this cycle
  ReferenceProcessor* rp = _g1h->ref_processor_cm();
  rp->start_discovery(false /* always_clear */);

  // 2. turn on SATB queue recording on every thread
  SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
  satb_mq_set.set_active_all_threads(true, /* new active value */ false /* expected_active */);

  // 3. mark the cycle in progress; start_concurrent_cycle (§4.1.2) then
  //    notifies G1CGC_lock to wake the marker thread
  cm_thread()->start_full_cycle();
}

// g1ConcurrentMark.cpp — one G1 Conc worker
void G1CMConcurrentMarkingTask::work(uint worker_id) {
  ResourceMark rm;
  SuspendibleThreadSetJoiner sts_join;   // a pause can suspend this worker
  G1CMTask* task = _cm->task(worker_id);
  task->record_start_time();
  do {
    task->do_marking_step(G1ConcMarkStepDurationMillis,
                          true  /* do_termination */,
                          false /* is_serial */);
    _cm->do_yield_check();               // yield if a pause was requested
  } while (!_cm->has_aborted() && task->has_aborted());
  task->record_end_time();
}

// g1ConcurrentMarkThread.cpp
void G1ConcurrentMarkThread::concurrent_mark_cycle_do() {
  // 1. this thread: scan roots of regions evacuated during Concurrent Start
  if (phase_scan_root_regions()) return;
  // 2. this thread drives the loop; G1 Conc workers run mark_from_roots
  if (phase_mark_loop()) return;
  // 3. this thread: rebuild remembered sets and scrub dead objects
  if (phase_rebuild_and_scrub()) return;
  if (phase_delay_to_keep_mmu_before_cleanup()) return;
  // 4. hand Cleanup to the VM thread
  if (phase_cleanup()) return;
  if (phase_clear_cld_claimed_marks()) return;
  // 5. this thread: clear the bitmap for the next cycle
  phase_clear_bitmap_for_next_mark();
}

// g1VMOperations.cpp — runs on the VM thread
void VM_G1CollectForAllocation::doit() {
  G1CollectedHeap* g1h = G1CollectedHeap::heap();
  GCCauseSetter x(g1h, _gc_cause);
  // 1. VM thread: evacuate the collection set (young, or young plus old candidates)
  g1h->do_collection_pause_at_safepoint(_word_size);

  if (_word_size > 0) {
    // 2. the allocation that triggered the pause retries, and may escalate
    _result = g1h->satisfy_failed_allocation(_word_size);
  } else if (g1h->should_upgrade_to_full_gc()) {
    // 3. no regions left: this same VM thread continues into a full collection
    g1h->upgrade_to_full_collection();
  }
}

// g1YoungCollector.cpp — one young (or mixed) pause (setup elided)
void G1YoungCollector::collect() {
  policy()->record_young_collection_start();   // tenuring threshold, pause timers

  pre_evacuate_collection_set(...);            // retire TLABs, select the collection set

  G1ParScanThreadStateSet per_thread_states(...);
  bool may_do_optional_evacuation = collection_set()->num_optional_regions() != 0;
  // The parallel copy: scan roots and remembered sets, copy live objects.
  evacuate_initial_collection_set(&per_thread_states, may_do_optional_evacuation);

  if (may_do_optional_evacuation) {
    evacuate_optional_collection_set(&per_thread_states);   // mixed: remaining old candidates
  }
  post_evacuate_collection_set(..., &per_thread_states);    // refs, free CSet, free list

  _next_state = policy()->record_young_collection_end(...); // statistics, next state
}

// g1VMOperations.cpp
void VM_G1CollectFull::doit() {
  G1CollectedHeap* g1h = G1CollectedHeap::heap();
  GCCauseSetter x(g1h, _gc_cause);
  bool clear_all_soft_refs = _gc_cause == GCCause::_metadata_GC_clear_soft_refs ||
                             _gc_cause == GCCause::_wb_full_gc;
  // VM thread: compact the whole heap
  g1h->do_full_collection(size_t(0) /* allocation_word_size */,
                          clear_all_soft_refs,
                          false /* do_maximal_compaction */);
}

// g1YoungGCPostEvacuateTasks.cpp
void G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask::do_work(uint worker_id) override {
  FreeCSetClosure cl(_surviving_young_words, worker_id,
                     worker_stats(worker_id), _evac_failure_regions);
  _g1h->collection_set_par_iterate_all(&cl, &_claimer, worker_id);
  ...
}

// g1CollectedHeap.cpp
void G1CollectedHeap::free_region(G1HeapRegion* hr, G1FreeRegionList* free_list) {
  // Reset region metadata to allow reuse.
  hr->hr_clear(true /* clear_space */);
  _policy->remset_tracker()->update_at_free(hr);
  if (free_list != nullptr) {
    free_list->add_ordered(hr);
  }
}

// g1CollectedHeap.cpp — end of the pause
void G1CollectedHeap::rebuild_free_region_list() {
  _hrm.rebuild_free_list(workers());
}

// g1IHOPControl.cpp
size_t G1IHOPControl::old_gen_threshold_for_conc_mark_start() const {
  if (!_is_adaptive || !have_enough_data_for_prediction()) {
    return (size_t)(_initial_ihop_percent * _target_occupancy / 100.0);
  }

  size_t old_non_humongous_alloc_bytes =
      (size_t)(predict(&_marking_start_to_mixed_time_s) * predict(&_old_non_humongous_alloc_rate));
  size_t predicted_peak_extra_humongous_occupancy =
      predict(&_peak_extra_humongous_occupancy_in_mark_cycle);
  size_t needed_for_concurrent_cycle = _expected_young_gen_at_first_mixed_gc
                                     + old_non_humongous_alloc_bytes
                                     + predicted_peak_extra_humongous_occupancy;

  size_t target_heap_occupancy = effective_target_occupancy();
  return needed_for_concurrent_cycle < target_heap_occupancy
           ? target_heap_occupancy - needed_for_concurrent_cycle
           : 0;
}

// g1Policy.cpp — the outer overload refuses during a concurrent
// cycle or when a mixed phase is about to start, then calls this one.
bool G1Policy::need_to_start_conc_mark(const char* source,
                                       const G1CollectorState& state,
                                       size_t allocation_word_size) const {
  size_t marking_initiating_old_gen_threshold =
      _ihop_control->old_gen_threshold_for_conc_mark_start();
  size_t non_young_occupancy =
      _g1h->non_young_occupancy_after_allocation(allocation_word_size);

  if (non_young_occupancy > marking_initiating_old_gen_threshold) {
    return state.is_in_young_only_phase();
  }
  return false;
}

// g1Policy.cpp — end of the young pause
G1CollectorState G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mark,
                                                       bool allocation_failure,
                                                       size_t allocation_word_size) {
  if (collector_state()->is_in_concurrent_start_gc()) {
    next_state.set_in_normal_young_gc();
  } else if (!about_to_start_mixed_phase() &&
             need_to_start_conc_mark("end of GC", next_state, allocation_word_size)) {
    // The flag the next pause reads.
    next_state.set_initiate_conc_mark_if_possible(true);
  }
}

// g1Policy.cpp
void G1Policy::decide_on_concurrent_start_pause() {
  if (collector_state()->initiate_conc_mark_if_possible()) {
    if (!about_to_start_mixed_phase() && collector_state()->is_in_young_only_phase()) {
      // This pause evacuates the young generation and then starts marking.
      collector_state()->set_in_concurrent_start_gc();
    }
  }
}

// g1CollectedHeap.cpp
HeapWord* G1CollectedHeap::attempt_allocation_humongous(size_t word_size) {
  if (is_init_completed() &&
      policy()->need_to_start_conc_mark("concurrent humongous allocation", word_size)) {
    try_collect(word_size, GCCause::_g1_humongous_allocation, collection_counters(this));
  }
}