AdsidacCallback.cpp

Go to the documentation of this file.

 
 
#include "AdsidacCallback.h"
#include "../Global/RestartTrim.h"
#include <cstdlib>
#include <filesystem>
#include "../Global/fmodule.h"
#include "../ParallelExec/ParallelGlobal.h"
#ifdef ENABLE_MPI
#include "../ParallelExec/ParallelExecution.h"
#endif
 
#include <iostream>
#include <fstream>
#include <memory>
#include <string>
#include <vector>
#include "../Global/SimulationContext.h"
#include "../Global/SharedConfig.h"
#include "../Global/CheckedIO.h"
 
using namespace std;
 
// Fortran membuf accessor. Not called in this file: onBranchingPoint obtains
// the restart buffer through solver.checkpoint().
extern "C" { void ads_membuf_get_(char** ptr, int* sz); }
 
extern void outputErrMessageToFile(string);
 
// ---- SolverCallback implementations ----
 
void AdsidacCallback::onEvaluationPoint(ThermalHydraulicSolver& solver) {
    BranchEndState *es_pt;
    string str;
    bool terminate_current_sequence = false;
 
    if (r5pars_.csnostop == 11) {
        es_pt = new BranchEndState();
        es_pt->setTypeOfConsequence(Consequence::THFail);
        str = "Plant model simulation failure";
        es_pt->setConsequenceDescription(str);
        g_sim.Branch_center.addAnEndState(es_pt);
        g_sim.expected_abort = true;
        return;
    } else if (r5pars_.csnostop != 0 && r5pars_.csnostop != 3) {
        es_pt = new BranchEndState();
        es_pt->setTypeOfConsequence(Consequence::THFail);
        str = "Sequence terminated by RELAP5";
        es_pt->setConsequenceDescription(str);
        g_sim.Branch_center.addAnEndState(es_pt);
        g_sim.expected_abort = true;
        return;
    }
    if (g_sim.fresh_restart) {
        g_sim.fresh_restart = false;
        initiationForRestart(solver);
        g_sim.current_seq_events.clear();
    }
    g_sim.Command_handler.sendInfoToCommunicationPlace();
    g_sim.Decision_Maker.scanControlPanel();
    g_sim.Action_Taker.scanControlPanel();
    g_sim.Indicator.setCurrentTime();
    g_sim.IE.checkAndExecuteInitiatingEvent();
    g_sim.Plant.updateParametersValues();
    g_sim.Indicator.oneThroughProcess();
    g_sim.Support_system_reliability.updateSupportSystemDisplayStates();
    g_sim.System_reliability.checkAndActiveComponentFailure();
    g_sim.Support_system_reliability.checkAndActiveSupportSystemFailure();
    g_sim.Plant.executeActionsToInteractWithSystem();
    g_sim.Decision_Maker.oneThroughProcessForBlockI();
    g_sim.Action_Taker.oneThroughProcessForBlockI();
    if (!g_sim.Decision_Maker.getPauseActivity()
        || !g_sim.Decision_Maker.getPauseMentalProcedure()) {
        g_sim.Decision_Maker.oneThroughProcessForBlockD();
    }
    if (!g_sim.Action_Taker.getPauseActivity()
        || !g_sim.Action_Taker.getPauseMentalProcedure()) {
        g_sim.Action_Taker.oneThroughProcessForBlockD();
    }
    g_sim.KPS.addDataSet();
 
    g_sim.Decision_Maker.updateEvidenceForAllPSFs();
    g_sim.Action_Taker.updateEvidenceForAllPSFs();
 
    terminate_current_sequence =
            g_sim.SScheduler.checkForTerminatingCurrentSequence();
    if (!terminate_current_sequence) {
        r5pars_.cnbranch = g_sim.Branch_center.checkAndAddBranchCluster();
        if (r5pars_.cnbranch > 1) {
            str = std::to_string(r5pars_.cnbranch);
            str += " branches are generated";
            g_sim.Aux_info.addRecord(str, -999);
            cout << "      " << r5pars_.timesno << "  " << str << endl;
            std::shared_ptr<RestartNodeInfo> restart_info_pt_1 = make_shared<RestartNodeInfo>();
            // Propagate the run's branch-ID epoch so peers receiving this
            // snapshot seed their windows in the same epoch.
            restart_info_pt_1->id_epoch_base = g_sim.id_epoch_base;
            restart_info_pt_1->setCorrespondingBranchClusterID(g_sim.Branch_center.getIDOfRecentGeneratedBranchCluster());
            restart_info_pt_1->saveRestartNodeInfo();
            int enq_cid = restart_info_pt_1->getCorrespondingBranchClusterID();
            g_sim.restart_info.push_back(restart_info_pt_1);
            g_sim.cid_to_source_sequence[enq_cid] = g_sim.Branch_center.getIDOfRecentGeneratedES() + 1;
            g_sim.branches_generated_this_sequence = true;
        }
        if (r5pars_.cnbranch > 0) {
            g_sim.Branch_center.sendCurrentBranchCommandToCommandCenter();
            r5pars_.cgennode = true;
        }
    } else {
        g_sim.Branch_center.clearBranches();
    }
    g_sim.Communication_place.clearCommunicationPlaceAllPointers();
}
 
void AdsidacCallback::onTimestepAction(ThermalHydraulicSolver& /*solver*/) {
    // Empty: the engine has no timestep-level crew actions
}
 
void AdsidacCallback::onScenarioStart(ThermalHydraulicSolver& /*solver*/) {
    g_sim.Plant.updateParametersValues();
    g_sim.Indicator.initiationForSimulation();
    g_sim.Support_system_reliability.updateSupportSystemDisplayStates();
}
 
void AdsidacCallback::onFirstInit(ThermalHydraulicSolver& /*solver*/) {
    g_sim.Indicator.readInfoFromFile();
    g_sim.All_procedure.initiationForSimulation();
    // Register the loaded procedure book as the process-local master.
    // Snapshots serialize state only; deserialization re-seeds structure and
    // definitions from this object (identical on every rank: same input
    // files, guarded by the config fingerprint).
    adsidac::set_shared_procedures(&g_sim.All_procedure);
    adsidac::set_shared_control_panel(&g_sim.Indicator);
    g_sim.All_polynomial.readInfoFromFile();
    g_sim.Decision_Maker.initiationForSimulation();
    g_sim.Action_Taker.initiationForSimulation();
    adsidac::set_shared_kb(sim::Operator::ODM, &g_sim.Decision_Maker.knowledge_base);
    adsidac::set_shared_kb(sim::Operator::OAT, &g_sim.Action_Taker.knowledge_base);
    adsidac::set_shared_diagnosis_matrix(sim::Operator::ODM, &g_sim.Decision_Maker.diagnosis_matrix);
    adsidac::set_shared_diagnosis_matrix(sim::Operator::OAT, &g_sim.Action_Taker.diagnosis_matrix);
    g_sim.System_reliability.readInfoFromFile();
    g_sim.Support_system_reliability.readInfoFromFile();
    g_sim.Indicator.initiationForSimulation();
    g_sim.Support_system_reliability.updateSupportSystemDisplayStates();
    g_sim.IE.InitiationForSimulation();
    g_sim.Decision_Maker.checkCompletenessOfSystemDecomposition(sim::Operator::ODM);
    g_sim.Action_Taker.checkCompletenessOfSystemDecomposition(sim::Operator::OAT);
    g_sim.Decision_Maker.scanControlPanel();
    g_sim.Action_Taker.scanControlPanel();
    g_sim.KPS.addDataSet();
}
 
void AdsidacCallback::onBranchingPoint(ThermalHydraulicSolver& solver) {
#ifdef ENABLE_MPI
    if (g_sim.config.is_mpi_environment)
    {
        if (PG::rank > 0 && g_client)
        {
            int local_id = g_client->getNextLocalBranchID();
            std::shared_ptr<RestartNodeInfo> rinfo = g_sim.restart_info.back();
            int cid = rinfo->getCorrespondingBranchClusterID();
 
            std::string branch_name = "restart_" + std::to_string(local_id)
                                    + "_cid" + std::to_string(cid) + ".r";
            bool use_mem = should_use_memory_restart();
            std::string local_branch_path = use_mem
                ? (g_sim.config.shm_prefix + "rank" + std::to_string(PG::rank) + "_" + branch_name)
                : ("../rank_" + std::to_string(PG::rank) + "/" + branch_name);
 
            auto ckpt_buf = solver.checkpoint();
 
            // Restart trimming (opt-in, see trim_enabled below): ship one
            // restart record, not the whole accumulated buffer.
            //
            // A captured buffer holds one dump per branch point for the entire
            // sequence. In MPI each branch is transferred separately and each
            // transfer carries all of it, while the receiver restores from
            // exactly one record: the one whose ncount goes into card 103,
            // which Client::parallel_restartforanewsequence_ reads from this
            // same cluster. So all N branches of this branch point want the
            // same single record.
            //
            // Placed here, before the memory/file fork, so it shrinks the held
            // buffer, the .r file AND the wire transfer together.
            // The whole block is MPI-only, so serial (which shares one buffer
            // across all children and needs every dump) is excluded by
            // construction.
            //
            // Falls back to the full buffer if the record is not found: extra
            // bandwidth is survivable, a wrong restart is not.
            BranchCluster* bc = g_sim.Branch_center.findBrnachCluster(cid);
            // Enabled only when the environment variable ADS_RESTART_TRIM is
            // "1" (read once per process); off by default.
            static const bool trim_enabled = [] {
                const char* v = std::getenv("ADS_RESTART_TRIM");
                return v && *v == '1';
            }();
            if (bc && !ckpt_buf.empty() && trim_enabled) {
                const auto want = static_cast<int32_t>(bc->getNCountForRestartRELAP5());
 
                // Publish the run-wide base once, so receivers can splice it
                // back on and the wire carries only [header][target]. The base
                // is byte-identical across lineages, which is what makes this
                // legal. It is written before the transfer that needs it, so
                // on a coherent shared filesystem a peer on another node sees
                // it. An existing restart_base.bin is left in place.
                static bool base_published = false;
                if (!base_published) {
                    const std::string base_path =
                        g_sim.SScheduler.getProjectDirectory() + "/restart_base.bin";
                    std::error_code ec;
                    if (!std::filesystem::exists(base_path, ec)) {
                        auto hb = adsidac::extract_header_and_base(ckpt_buf);
                        if (!hb.empty()) {
                            // temp + rename: several ranks may reach this at once
                            const std::string tmp = base_path + ".r"
                                + std::to_string(PG::rank) + ".tmp";
                            write_file_or_die(tmp, hb.data(), hb.size());
                            std::filesystem::rename(tmp, base_path, ec);
                            if (ec) std::filesystem::remove(tmp, ec);
                        }
                    }
                    base_published = true;
                }
 
                auto one = adsidac::extract_record_no_base(ckpt_buf, want);
                if (!one.empty()) {
                    if (const char* t = std::getenv("ADS_MEMBUF_TRACE"); t && *t == '1') {
                        std::cout << "[TRIM] cid=" << cid << " ncount=" << want
                                  << "  " << ckpt_buf.size() / 1048576.0 << " MB -> "
                                  << one.size() / 1048576.0 << " MB  ("
                                  << (double)ckpt_buf.size() / one.size() << "x)" << std::endl;
                    }
                    ckpt_buf = std::move(one);
                } else {
                    PU::print_message(PG::rank, "[TRIM] cid=" + std::to_string(cid)
                        + " ncount=" + std::to_string(want)
                        + " not found in buffer — sending it whole", PG::log);
                }
            }
 
            if (use_mem) {
                if (!ckpt_buf.empty() && !g_sim.restart_info.empty()) {
                    if (g_sim.config.compress_queue) {
                        rinfo->relap_restart_buf = lz4_compress(ckpt_buf.data(), static_cast<int>(ckpt_buf.size()));
                    } else {
                        rinfo->relap_restart_buf = std::move(ckpt_buf);
                    }
                }
            } else {
                if (!ckpt_buf.empty()) {
                    if (g_sim.config.compress_queue) {
                        auto comp = lz4_compress(ckpt_buf.data(), static_cast<int>(ckpt_buf.size()));
                        write_file_or_die(local_branch_path, comp.data(), comp.size());
                    } else {
                        write_file_or_die(local_branch_path, ckpt_buf.data(), ckpt_buf.size());
                    }
                }
            }
 
            int num_branches = static_cast<int>(bc->all_branch.size());
 
            PU::print_message(PG::rank, "[PEER] CID=" + std::to_string(cid)
                + " branches=" + std::to_string(num_branches)
                + " RSS=" + std::to_string(get_rss_mb()) + "MB"
                + " buf=" + std::to_string(rinfo->relap_restart_buf.size()/1024/1024) + "MB", PG::log);
 
            MPI_Send(&local_id, 1, MPI_INT, 0, PG::server_branch_meta, MPI_COMM_WORLD);
            MPI_Send(&num_branches, 1, MPI_INT, 0, PG::server_branch_meta, MPI_COMM_WORLD);
 
            int num_direct;
            MPI_Recv(&num_direct, 1, MPI_INT, 0, PG::client_peer_assign,
                     MPI_COMM_WORLD, MPI_STATUS_IGNORE);
 
            // Serialize the snapshot once and send the same bytes to every
            // direct target: rinfo embeds the restart buffer and BranchRecord
            // grows with the tree, so per-target serialization is costly.
            std::string rinfo_bytes, branch_bytes;
            if (num_direct > 0) {
                rinfo_bytes = PU::serialize_object(*rinfo);
                branch_bytes = PU::serialize_object(g_sim.Branch_center);
            }
 
            for (int i = 0; i < num_direct; i++) {
                int target_rank, branch_index, seq_id, parent_seq_id;
                MPI_Recv(&target_rank, 1, MPI_INT, 0, PG::client_peer_assign,
                         MPI_COMM_WORLD, MPI_STATUS_IGNORE);
                MPI_Recv(&branch_index, 1, MPI_INT, 0, PG::client_peer_assign,
                         MPI_COMM_WORLD, MPI_STATUS_IGNORE);
                MPI_Recv(&seq_id, 1, MPI_INT, 0, PG::client_peer_assign,
                         MPI_COMM_WORLD, MPI_STATUS_IGNORE);
                MPI_Recv(&parent_seq_id, 1, MPI_INT, 0, PG::client_peer_assign,
                         MPI_COMM_WORLD, MPI_STATUS_IGNORE);
 
                PU::print_message(PG::rank, "[PEER] Direct send -> C["
                    + std::to_string(target_rank) + "] bi="
                    + std::to_string(branch_index) + " seq=" + std::to_string(seq_id), PG::log);
 
                PU::send_bytes(rinfo_bytes, target_rank, PG::client_peer_restart, MPI_COMM_WORLD);
                PU::send_bytes(branch_bytes, target_rank, PG::client_peer_restart, MPI_COMM_WORLD);
                int path_len = static_cast<int>(local_branch_path.size());
                MPI_Send(&path_len, 1, MPI_INT, target_rank, PG::client_peer_restart, MPI_COMM_WORLD);
                MPI_Send(local_branch_path.c_str(), path_len, MPI_CHAR,
                         target_rank, PG::client_peer_restart, MPI_COMM_WORLD);
                MPI_Send(&seq_id, 1, MPI_INT, target_rank, PG::client_peer_restart, MPI_COMM_WORLD);
                MPI_Send(&branch_index, 1, MPI_INT, target_rank, PG::client_peer_restart, MPI_COMM_WORLD);
                MPI_Send(&parent_seq_id, 1, MPI_INT, target_rank, PG::client_peer_restart, MPI_COMM_WORLD);
            }
 
            int num_queued;
            MPI_Recv(&num_queued, 1, MPI_INT, 0, PG::client_peer_assign,
                     MPI_COMM_WORLD, MPI_STATUS_IGNORE);
 
            if (num_queued > 0) {
                g_client->hold_branch_data(local_id, rinfo, g_sim.Branch_center, local_branch_path, num_queued);
            } else {
                // Every branch was sent directly to a peer, so nothing holds
                // this snapshot, but the shared_ptr still in
                // g_sim.restart_info would keep the restart buffer alive until
                // the next task clears the vector. Release the buffer now.
                rinfo->relap_restart_buf.clear();
                rinfo->relap_restart_buf.shrink_to_fit();
            }
 
            g_client->branches_this_seq_++;
 
            PU::print_message(PG::rank, "[PEER] Branch done: "
                + std::to_string(num_direct) + " direct, "
                + std::to_string(num_queued) + " queued (held locally)", PG::log);
        }
    }
#endif
}
 
// ---- Restart management ----
 
void AdsidacCallback::restartForNewSequence(ThermalHydraulicSolver& solver) {
    bool found_restart_point = false;
    long n_count;
    string content = "100 restart  transnt\n101 run\n103 ";
 
    size_t chosen_idx = static_cast<size_t>(-1);
    const size_t total = g_sim.Branch_center.branch_cluster.size();
    for (size_t idx = 0; idx < total; ++idx) {
        BranchCluster *bc = g_sim.Branch_center.branch_cluster[idx].get();
        if (bc->getCurrentBranchNum() < (static_cast<int>(bc->all_branch.size()) - 1)) {
            chosen_idx = idx;
            break;
        }
    }
 
    if (chosen_idx != static_cast<size_t>(-1)) {
        BranchCluster *chosen = g_sim.Branch_center.branch_cluster[chosen_idx].get();
        int cid = chosen->getIDOfThisBranchCluster();
        n_count = chosen->getNCountForRestartRELAP5();
        auto seq_it = g_sim.cid_to_source_sequence.find(cid);
        if (seq_it != g_sim.cid_to_source_sequence.end()) {
            auto buf_it = g_sim.restart_buffers.find(seq_it->second);
            if (buf_it != g_sim.restart_buffers.end() && !buf_it->second.empty()) {
                solver.restore(buf_it->second);
            } else {
                std::string backup = restart_path("restart_seq_" + std::to_string(seq_it->second) + ".r");
                std::ifstream ifs(backup, std::ios::binary | std::ios::ate);
                if (!ifs || ifs.tellg() <= 0) {
                    std::cerr << "ERROR: cannot read restart file: " << backup << std::endl;
                } else {
                    auto file_sz = static_cast<size_t>(ifs.tellg());
                    ifs.seekg(0);
                    std::vector<char> buf(file_sz);
                    ifs.read(buf.data(), static_cast<std::streamsize>(file_sz));
                    solver.restore(buf);
                }
            }
        }
 
        found_restart_point = true;
        g_sim.fresh_restart = true;
    } else {
        solver.setTerminateAll(true);
        for (auto &bc : g_sim.Branch_center.branch_cluster) {
            bc->savePENodeToFile();
        }
        g_sim.Branch_center.branch_cluster.clear();
 
        for (auto& [c, s] : g_sim.cid_to_source_sequence) {
            std::string bkp = restart_path("restart_seq_" + std::to_string(s) + ".r");
            if (fs::exists(bkp)) fs::remove(bkp);
        }
        g_sim.cid_to_source_sequence.clear();
 
        cout << "All sequences have been simulated" << endl;
    }
 
    if (found_restart_point) {
        content += to_string(n_count);
        content += "\n.";
        ofstream fout("indeck.i", ios::out);
        fout << content;
        fout.close();
    } else if (!solver.isTerminateAll()) {
        ofstream fout("err messages.txt", ios::app);
        cout << "restartForANewSequence: could not find the restart point."
             << endl;
        fout << r5pars_.timesno
             << "  restartForANewSequence: could not find the restart point.\n";
        fout.close();
    }
 
    g_sim.KPS.clearKeyParameterHistory();
    g_sim.Aux_info.clearAnalysisInfoContainerForRestart();
}
 
void AdsidacCallback::initiationForRestart(ThermalHydraulicSolver& solver) {
    bool found_restart_branch = false;
    bool found_restart_info = false;
    int restart_branch_cluster_ID = -1, restart_branch_num = -1, new_branch_ID;
    int id_of_mother_branch_cluster, id_of_mother_branch;
    // Assigned from g_sim.restart_info when found; only dereferenced inside
    // if (found_restart_info). Left null here because constructing a
    // RestartNodeInfo builds two crew BBNs.
    std::shared_ptr<RestartNodeInfo> restart_info_pt;
    BranchProbability *restart_branch_pt = nullptr;
    BranchCluster *mother_bc_pt;
    BranchProbability *mother_branch;
 
    for (size_t idx = 0; idx < g_sim.Branch_center.branch_cluster.size(); ++idx) {
        BranchCluster *bc = g_sim.Branch_center.branch_cluster[idx].get();
        int cid = bc->getIDOfThisBranchCluster();
        int curr = bc->getCurrentBranchNum();
        if (curr < (static_cast<int>(bc->all_branch.size()) - 1)) {
            found_restart_branch = true;
            restart_branch_cluster_ID = cid;
            restart_branch_num = curr + 1;
            g_sim.seq_restart_cid = cid;
            g_sim.seq_restart_branch_num = restart_branch_num;
            bc->setCurrentBranchNum(restart_branch_num);
            new_branch_ID = g_sim.Branch_center.getIDOfRecentGeneratedBranch() + 1;
            g_sim.Branch_center.setIDOfRecentGeneratedBranch(new_branch_ID);
            restart_branch_pt = bc->all_branch.at(restart_branch_num).get();
            restart_branch_pt->setIDOfThisBranch(new_branch_ID);
            g_sim.Branch_center.setIDOfMotherCluster(bc->getIDOfThisBranchCluster());
            g_sim.Branch_center.setIDOfMotherBranch(new_branch_ID);
            g_sim.Branch_center.setRecentUnconditionalBranchProbability(
                    restart_branch_pt->getUnconditionalBranchProbability());
 
            id_of_mother_branch_cluster = bc->getIDOfMotherCluster();
            id_of_mother_branch = bc->getIDOfTheMotherBranch();
            mother_bc_pt = g_sim.Branch_center.findBrnachCluster(id_of_mother_branch_cluster);
            if (mother_bc_pt) {
                mother_branch = mother_bc_pt->findBranch(id_of_mother_branch);
                if (mother_branch) {
                    mother_branch->child_IDs.push_back(new_branch_ID);
                }
            }
            break;
        }
    }
 
    if (found_restart_branch) {
        bool cid_consumed = false;
        for (auto ri = g_sim.restart_info.begin(); ri != g_sim.restart_info.end(); ++ri) {
            if (*ri && (*ri)->getCorrespondingBranchClusterID() == restart_branch_cluster_ID) {
                found_restart_info = true;
                restart_info_pt = *ri;
 
                BranchCluster *bc = g_sim.Branch_center.findBrnachCluster(restart_branch_cluster_ID);
                cid_consumed = (bc && restart_branch_num >= static_cast<int>(bc->all_branch.size()) - 1);
 
                if (cid_consumed) {
                    *ri = nullptr;
                    g_sim.restart_info.erase(ri);
                }
                break;
            }
        }
 
        if (found_restart_info) {
            g_sim.Indicator.clearForRestart();
            g_sim.Communication_place.clearCommunicationPlaceForRestart();
            g_sim.Command_handler.clearCommandControlCenteForRestart();
            g_sim.All_procedure.clearProceduresForRestart();
            g_sim.Aux_info.clearAnalysisInfoContainerForRestart();
            g_sim.KPS.clearKeyParameterHistory();
            g_sim.Decision_Maker.clearOperatorDMForRestart();
            g_sim.Action_Taker.clearOperatorATForRestart();
            g_sim.IE.clearInitiatingEventController();
            g_sim.Branch_center.clearBranchRecordForRestart();
            g_sim.System_reliability.clearForRestart();
            g_sim.Support_system_reliability.clearForRestart();
            g_sim.All_polynomial.clearCalculationAidForRestart();
            g_sim.Decision_Maker.m_RM.reset();
            g_sim.Action_Taker.m_RM.reset();
 
            if (g_sim.Plant.setConnectionToRELAP5() == -1) {
                // Card-number remapping failed mid-run: continuing would read
                // stale _obj indices. Flag the sequence (AdsSimErr end state)
                // instead of simulating from a broken bridge.
                r5pars_.sim_err = true;
                g_sim.Branch_center.setTempESMessage("setConnectionToRELAP5 failed at restart.");
                g_sim.Branch_center.setTempTypeOfConsequence(Consequence::AdsSimErr);
            }
            g_sim.Indicator.copyControlPanelForRestart(restart_info_pt->r_Indicator);
            g_sim.All_procedure.copyProceduresFromRestartInfoForRestart(
                    restart_info_pt->r_All_procedure);
            g_sim.Aux_info.copyAnalysisInfoContainerForRestart(
                    restart_info_pt->r_Aux_info);
            g_sim.KPS.copyKeyParameterHistory(restart_info_pt->r_KPS);
            g_sim.Command_handler.copyCommandControlCenterFromRestartInfo(
                    restart_info_pt->r_Command_handler);
            g_sim.Decision_Maker.copyOperatorDMFromRestartInfo(
                    restart_info_pt->r_Decision_Maker);
            g_sim.Action_Taker.copyOperatorATFromRestartInfo(
                    restart_info_pt->r_Action_Taker);
            g_sim.IE.copyInitiatingEventController(restart_info_pt->r_IE);
            g_sim.System_reliability.copyHWReliabilityModule(
                    restart_info_pt->r_System_reliability);
            g_sim.All_polynomial.copyCalculationAidFromRestartInfoForRestart(
                    restart_info_pt->r_All_polynomial);
 
            // Give the selected branch its content. The snapshot is taken
            // BEFORE the mainline (branch 0) is injected, so the restored
            // command center holds only the commands pending at the branch
            // point; the alternative this child stands for is loaded here.
            // Environment variable ADS_RESTART_INJECT (read once per process):
            // "0" skips the injection; unset or any other value injects
            // (the default).
            static const bool inject_on_restart = [] {
                const char* v = std::getenv("ADS_RESTART_INJECT");
                return !(v && *v == '0');
            }();
            if (inject_on_restart) {
                restart_branch_pt->sendBranchInfoToCommandCenter(new_branch_ID);
            }
 
            if (cid_consumed) {
                auto cs_it = g_sim.cid_to_source_sequence.find(restart_branch_cluster_ID);
                if (cs_it != g_sim.cid_to_source_sequence.end()) {
                    int consumed_seq = cs_it->second;
                    g_sim.cid_to_source_sequence.erase(cs_it);
                    bool still_needed = false;
                    for (auto& [c, s] : g_sim.cid_to_source_sequence) {
                        if (s == consumed_seq) { still_needed = true; break; }
                    }
                    if (!still_needed) {
                        std::string bkp = restart_path("restart_seq_" + std::to_string(consumed_seq) + ".r");
                        if (fs::exists(bkp)) {
                            fs::remove(bkp);
                        }
                        g_sim.restart_buffers.erase(consumed_seq);
                    }
                }
            }
        } else {
            ofstream fout("err messages.txt", ios::app);
            cout << r5pars_.timesno
                 << "\tADSV2::initiationForRestart: found restart branch but not restart info node."
                 << endl;
            fout << r5pars_.timesno
                 << "\tADSV2::initiationForRestart: found restart branch but not restart info node.\n";
            fout.close();
        }
    } else {
        ofstream fout("err messages.txt", ios::app);
        cout << r5pars_.timesno
             << "\tADSV2::initiationForRestart: count not find restart branch."
             << endl;
        fout << r5pars_.timesno
             << "\tADSV2::initiationForRestart: count not find restart branch.\n";
        fout.close();
        solver.setTerminateAll(true);
    }
}