diff --git a/env/bash b/env/bash index 8c9c2abd..189f76bd 100644 --- a/env/bash +++ b/env/bash @@ -69,7 +69,7 @@ if [[ $PARTITION == "unknown" ]]; then elif [[ $PARTITION == "chicoma-gpu" ]]; then module unload cray-libsci module load PrgEnv-gnu - module load cudatoolkit/24.7_12.5 + module load cudatoolkit export CRAY_ACCEL_TARGET=nvidia80 export MPICH_GPU_SUPPORT_ENABLED=1 export MPICH_GPU_MANAGED_MEMORY_SUPPORT_ENABLED=1 diff --git a/src/artemis_driver.cpp b/src/artemis_driver.cpp index bb001c7b..41bceb71 100644 --- a/src/artemis_driver.cpp +++ b/src/artemis_driver.cpp @@ -114,6 +114,7 @@ ArtemisDriver::ArtemisDriver(ParameterInput *pin, ApplicationInput *app_in //! \brief Assembles the tasks associated with a step for the ArtemisDriver template TaskListStatus ArtemisDriver::Step() { + PARTHENON_INSTRUMENT // Prepare registers PreStepTasks(); TaskListStatus status = TaskListStatus::complete; @@ -158,6 +159,7 @@ TaskListStatus ArtemisDriver::Step() { //! \brief Defines the tasks executed prior to the main integrator in the ArtemisDriver template void ArtemisDriver::PreStepTasks() { + PARTHENON_INSTRUMENT // set the integration timestep integrator->dt = tm.dt; if (do_nbody) nbody_integrator->dt = tm.dt; @@ -194,6 +196,7 @@ void ArtemisDriver::PreStepTasks() { //! \brief Defines the main integrator's TaskCollection for the ArtemisDriver template TaskCollection ArtemisDriver::StepTasks() { + PARTHENON_INSTRUMENT using TQ = TaskQualifier; using namespace ::parthenon::Update; TaskCollection tc; @@ -338,6 +341,7 @@ TaskCollection ArtemisDriver::StepTasks() { //! \brief Defines the TaskCollection for post step tasks in the ArtemisDriver template TaskCollection ArtemisDriver::PostStepTasks() { + PARTHENON_INSTRUMENT using namespace ::parthenon::Update; TaskCollection tc; TaskID none(0); diff --git a/src/derived/fill_derived.cpp b/src/derived/fill_derived.cpp index 2ca22eab..787eefb9 100644 --- a/src/derived/fill_derived.cpp +++ b/src/derived/fill_derived.cpp @@ -29,6 +29,7 @@ namespace ArtemisDerived { //! NOTE(PDM): Note that this function is not called during remeshing. template TaskStatus SetAuxillaryFields(MeshData *md) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -94,6 +95,7 @@ TaskStatus SetAuxillaryFields(MeshData *md) { //! or remeshing event in preparation for FillGhost template void ConsToPrim(MeshData *md) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -224,6 +226,7 @@ void ConsToPrim(MeshData *md) { //! \brief Executes P2C following integrator updates and/or remeshing events template void PrimToCons(T *md) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -383,6 +386,7 @@ void PrimToCons(T *md) { //! but before PreCommFillDerived template void PostInitialization(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT auto &md = pmb->meshblock_data.Get(); PrimToCons, GEOM>(md.get()); } @@ -392,6 +396,7 @@ void PostInitialization(MeshBlock *pmb, ParameterInput *pin) { //! \brief Syncs unsplit fields following an operator split update template TaskCollection SyncFields(Mesh *pmesh, const Real time, const Real dt) { + PARTHENON_INSTRUMENT using namespace ::parthenon::Update; TaskCollection tc; TaskID none(0); diff --git a/src/drag/drag.cpp b/src/drag/drag.cpp index fb5491f0..c1ce0255 100644 --- a/src/drag/drag.cpp +++ b/src/drag/drag.cpp @@ -89,6 +89,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { //! \brief Calls source terms for drag template TaskStatus DragSource(MeshData *md, const Real time, const Real dt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; auto pm = md->GetParentPointer(); diff --git a/src/drag/drag.hpp b/src/drag/drag.hpp index baa69247..4358cdd8 100644 --- a/src/drag/drag.hpp +++ b/src/drag/drag.hpp @@ -164,6 +164,7 @@ template TaskStatus SelfDragSourceImpl(MeshData *md, const Real time, const Real dt, const Diffusion::DiffCoeffParams &dp, const EOS &eos_d, const SelfDragParams &gasp, const SelfDragParams &dustp) { + PARTHENON_INSTRUMENT using TE = parthenon::TopologicalElement; auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -344,6 +345,7 @@ TaskStatus SimpleDragSourceImpl(MeshData *md, const Real time, const Real const Diffusion::DiffCoeffParams &dp, const EOS &eos_d, const SelfDragParams &gasp, const SelfDragParams &dustp, const StoppingTimeParams &tp) { + PARTHENON_INSTRUMENT using TE = parthenon::TopologicalElement; auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; diff --git a/src/dust/dust.cpp b/src/dust/dust.cpp index be1df4be..9e5aa5c4 100644 --- a/src/dust/dust.cpp +++ b/src/dust/dust.cpp @@ -223,6 +223,7 @@ std::shared_ptr Initialize(ParameterInput *pin, //! \brief Compute dust hydrodynamics timestep template Real EstimateTimestepMesh(MeshData *md) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using RotatingFrame::BackgroundVelocity; auto pm = md->GetParentPointer(); @@ -271,6 +272,7 @@ Real EstimateTimestepMesh(MeshData *md) { //! \fn TaskStatus Dust::CalculateFluxes //! \brief Evaluates advective fluxes for dust evolution TaskStatus CalculateFluxes(MeshData *md, const bool pcm) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -300,6 +302,7 @@ TaskStatus CalculateFluxes(MeshData *md, const bool pcm) { //! \fn TaskStatus Dust::FluxSource //! \brief Evaluates coordinate terms from advective fluxes for dust evolution TaskStatus FluxSource(MeshData *md, const Real dt) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -336,6 +339,7 @@ TaskStatus FluxSource(MeshData *md, const Real dt) { //! \brief Add history outputs for dust quantities for generic coordinate system template void AddHistoryImpl(Params ¶ms) { + PARTHENON_INSTRUMENT using namespace ArtemisUtils; auto HstSum = parthenon::UserHistoryOperation::sum; using parthenon::HistoryOutputVar; @@ -361,6 +365,7 @@ void AddHistoryImpl(Params ¶ms) { //! \fn void Dust::AddHistory //! \brief Add history outputs for dust quantities void AddHistory(Coordinates coords, Params ¶ms) { + PARTHENON_INSTRUMENT if (coords == Coordinates::cartesian) { AddHistoryImpl(params); } else if (coords == Coordinates::cylindrical) { diff --git a/src/gas/cooling/beta_cooling.cpp b/src/gas/cooling/beta_cooling.cpp index 6a82792a..b1aa6e36 100644 --- a/src/gas/cooling/beta_cooling.cpp +++ b/src/gas/cooling/beta_cooling.cpp @@ -39,6 +39,7 @@ namespace Cooling { //! Tp - T = -om dt (T - T0) / (beta + om dt) template TaskStatus BetaCooling(MeshData *md, const Real time, const Real dt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pm = md->GetParentPointer(); diff --git a/src/gas/cooling/cooling.cpp b/src/gas/cooling/cooling.cpp index f59cdefe..c1ebf031 100644 --- a/src/gas/cooling/cooling.cpp +++ b/src/gas/cooling/cooling.cpp @@ -90,6 +90,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { //! \brief Wrapper function for external cooling options template TaskStatus CoolingSource(MeshData *md, const Real time, const Real dt) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &pkg = pm->packages.Get("cooling"); CoolingType ctype = pkg->template Param("type"); diff --git a/src/gas/gas.cpp b/src/gas/gas.cpp index 63199ace..fd33b4f6 100644 --- a/src/gas/gas.cpp +++ b/src/gas/gas.cpp @@ -439,6 +439,7 @@ std::shared_ptr Initialize(ParameterInput *pin, //! \brief Compute gas hydrodynamics timestep template Real EstimateTimestepMesh(MeshData *md) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using RotatingFrame::BackgroundVelocity; auto pm = md->GetParentPointer(); @@ -527,6 +528,7 @@ Real EstimateTimestepMesh(MeshData *md) { //! \fn TaskStatus Gas::CalculateFluxes //! \brief Evaluates advective fluxes for gas evolution TaskStatus CalculateFluxes(MeshData *md, const bool pcm) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -559,6 +561,7 @@ TaskStatus CalculateFluxes(MeshData *md, const bool pcm) { //! \fn TaskStatus Gas::FluxSource //! \brief Evaluates coordinate terms from advective fluxes for gas evolution TaskStatus FluxSource(MeshData *md, const Real dt) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -590,6 +593,7 @@ TaskStatus FluxSource(MeshData *md, const Real dt) { // \brief Evaluates viscous flux template TaskStatus ViscousFlux(MeshData *md) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &pkg = pm->packages.Get("gas"); @@ -629,6 +633,7 @@ TaskStatus ViscousFlux(MeshData *md) { // \brief Evaluates thermal flux template TaskStatus ThermalFlux(MeshData *md) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &pkg = pm->packages.Get("gas"); @@ -666,6 +671,7 @@ TaskStatus ThermalFlux(MeshData *md) { //! \fn TaskStatus Gas::ZeroDiffusionFlux // \brief Resets the diffusion flux TaskStatus ZeroDiffusionFlux(MeshData *md) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &pkg = pm->packages.Get("gas"); @@ -683,6 +689,7 @@ TaskStatus ZeroDiffusionFlux(MeshData *md) { // \brief Applies the diffusion fluxes to update the momenta and energy template TaskStatus DiffusionUpdate(MeshData *md, const Real dt) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &pkg = pm->packages.Get("gas"); @@ -709,6 +716,7 @@ TaskStatus DiffusionUpdate(MeshData *md, const Real dt) { } TaskStatus DepositEnergy(MeshData *md, const Real dt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -739,6 +747,7 @@ TaskStatus DepositEnergy(MeshData *md, const Real dt) { //! \brief Add history outputs for gas quantities for generic coordinate system template void AddHistoryImpl(Params ¶ms) { + PARTHENON_INSTRUMENT using namespace ArtemisUtils; auto HstSum = parthenon::UserHistoryOperation::sum; using parthenon::HistoryOutputVar; @@ -772,6 +781,7 @@ void AddHistoryImpl(Params ¶ms) { //! \fn void Gas::AddHistory //! \brief Add history outputs for gas quantities void AddHistory(Coordinates coords, Params ¶ms) { + PARTHENON_INSTRUMENT if (coords == Coordinates::cartesian) { AddHistoryImpl(params); } else if (coords == Coordinates::cylindrical) { diff --git a/src/geometry/geometry.cpp b/src/geometry/geometry.cpp index d1e700c2..bf896980 100644 --- a/src/geometry/geometry.cpp +++ b/src/geometry/geometry.cpp @@ -106,6 +106,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { //! \brief Initializes the geometry meshblock data template void InitBlockGeom(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; auto pm = pmb->pmy_mesh; const int ndim_ = pm->ndim; diff --git a/src/gravity/binary_mass.cpp b/src/gravity/binary_mass.cpp index 44c97b3b..f94074dc 100644 --- a/src/gravity/binary_mass.cpp +++ b/src/gravity/binary_mass.cpp @@ -25,6 +25,7 @@ namespace Gravity { //! \brief Applies accelerations due to a binary template TaskStatus BinaryMassGravity(MeshData *md, const Real time, const Real dt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pm = md->GetParentPointer(); diff --git a/src/gravity/gravity.cpp b/src/gravity/gravity.cpp index 03aaa9bd..006e38b0 100644 --- a/src/gravity/gravity.cpp +++ b/src/gravity/gravity.cpp @@ -133,6 +133,7 @@ std::shared_ptr Initialize(ParameterInput *pin, //! \brief Wrapper function for external gravity options template TaskStatus ExternalGravity(MeshData *md, const Real time, const Real dt) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &pkg = pm->packages.Get("gravity"); diff --git a/src/gravity/nbody_gravity.hpp b/src/gravity/nbody_gravity.hpp index e4ab602d..9b18b255 100644 --- a/src/gravity/nbody_gravity.hpp +++ b/src/gravity/nbody_gravity.hpp @@ -162,6 +162,7 @@ NBodyGravityImpl(V1 &vmesh, V2 &vg, const geometry::Coords &coords, //! \brief Applies accelerations due to collection of point masses template TaskStatus NBodyGravity(MeshData *md, const Real time, const Real dt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pm = md->GetParentPointer(); diff --git a/src/gravity/point_mass.cpp b/src/gravity/point_mass.cpp index e0f99f56..77dd50bf 100644 --- a/src/gravity/point_mass.cpp +++ b/src/gravity/point_mass.cpp @@ -25,6 +25,7 @@ namespace Gravity { //! \brief Applies accelerations due to a point mass gravitational potential template TaskStatus PointMassGravity(MeshData *md, const Real time, const Real dt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pm = md->GetParentPointer(); diff --git a/src/gravity/uniform.cpp b/src/gravity/uniform.cpp index 3d840a6f..408a3186 100644 --- a/src/gravity/uniform.cpp +++ b/src/gravity/uniform.cpp @@ -26,6 +26,7 @@ namespace Gravity { //! \brief Applies accelerations due to a constant g template TaskStatus UniformGravity(MeshData *md, const Real time, const Real dt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pm = md->GetParentPointer(); diff --git a/src/main.cpp b/src/main.cpp index 11034a22..28bee513 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -24,6 +24,7 @@ parthenon::DriverStatus LaunchWorkFlow(parthenon::ParthenonManager &pman, parthenon::ParameterInput *pin) { + PARTHENON_INSTRUMENT // Geometry specific routines // (1) Handle ProblemGenerators and associated modifiers // (2) Call ParthenonInit to set up the mesh and packages diff --git a/src/nbody/nbody.cpp b/src/nbody/nbody.cpp index 3a73ec4c..af8c6abb 100644 --- a/src/nbody/nbody.cpp +++ b/src/nbody/nbody.cpp @@ -261,6 +261,7 @@ std::shared_ptr Initialize(ParameterInput *pin, //! \fn Real NBody::EstimateTimestepMesh //! \brief Compute NBody timestep Real EstimateTimestepMesh(MeshData *md) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; auto pm = md->GetParentPointer(); @@ -274,6 +275,7 @@ Real EstimateTimestepMesh(MeshData *md) { //! \brief Distance based refinement criterion template AmrTag DistanceRefinement(MeshBlockData *md) { + PARTHENON_INSTRUMENT auto pmb = md->GetBlockPointer(); auto pm = pmb->pmy_mesh; auto &resolved_pkgs = pm->resolved_packages; @@ -324,6 +326,7 @@ AmrTag DistanceRefinement(MeshBlockData *md) { //! \brief Create REBOUND restart file and store in Params to reuse as Parthenon restart void UserWorkBeforeRestartOutputMesh(Mesh *pmesh, ParameterInput *, SimTime &, OutputParameters *) { + PARTHENON_INSTRUMENT auto &artemis_pkg = pmesh->packages.Get("artemis"); if (!(artemis_pkg->Param("do_nbody"))) return; diff --git a/src/nbody/nbody_advance.cpp b/src/nbody/nbody_advance.cpp index 35c5043b..3a571491 100644 --- a/src/nbody/nbody_advance.cpp +++ b/src/nbody/nbody_advance.cpp @@ -84,6 +84,7 @@ namespace NBody { //! stage 2 integrates to end TaskStatus Advance(Mesh *pm, const Real time, const int stage, const parthenon::LowStorageIntegrator *nbody_integ) { + PARTHENON_INSTRUMENT auto &nbody_pkg = pm->packages.Get("nbody"); auto particle_id = nbody_pkg->Param>("particle_id"); auto particles = nbody_pkg->Param>("particles"); diff --git a/src/nbody/nbody_outputs.cpp b/src/nbody/nbody_outputs.cpp index 752e3d76..6a253849 100644 --- a/src/nbody/nbody_outputs.cpp +++ b/src/nbody/nbody_outputs.cpp @@ -30,6 +30,7 @@ namespace NBody { //! \fn void NBody::Outputs //! \brief void Outputs(parthenon::Mesh *pm, const Real time) { + PARTHENON_INSTRUMENT // Return immediately if nbody outputs are disabled auto nbody = pm->packages.Get("nbody").get(); if (nbody->Param("disable_outputs")) return; diff --git a/src/nbody/nbody_utils.hpp b/src/nbody/nbody_utils.hpp index 344551f6..b44d246c 100644 --- a/src/nbody/nbody_utils.hpp +++ b/src/nbody/nbody_utils.hpp @@ -122,6 +122,7 @@ static void PrintSystem(const int npart, ParArray1D particles) { //! \brief Copy the positions and velocities from the rebound sim to the Particles list static void SyncWithRebound(RebSim &r_sim, std::vector particle_id, ParArray1D particles) { + PARTHENON_INSTRUMENT const auto npart = particle_id.size(); auto particles_h = particles.GetHostMirrorAndCopy(); @@ -211,6 +212,7 @@ static void enable_stderr(int stderr_save_fd) { //! \fn int NBody::write_bytes_to_file //! \brief inline void write_bytes_to_file(std::string filename, std::vector &bytes) { + PARTHENON_INSTRUMENT std::ofstream outfile(filename.c_str(), std::ios::binary); if (outfile.is_open()) { outfile.write(reinterpret_cast(bytes.data()), bytes.size()); @@ -224,6 +226,7 @@ inline void write_bytes_to_file(std::string filename, std::vector &bytes) //! \fn int NBody::read_bytes_from_file //! \brief inline std::vector read_bytes_from_file(std::string filename) { + PARTHENON_INSTRUMENT std::ifstream file(NBody::rebound_filename, std::ios::binary | std::ios::ate); if (file.is_open()) { auto size = file.tellg(); diff --git a/src/pgen/advection.hpp b/src/pgen/advection.hpp index b93eb2f8..520cfa84 100644 --- a/src/pgen/advection.hpp +++ b/src/pgen/advection.hpp @@ -62,6 +62,7 @@ static AdvectionVariables av; //! \brief Sets initial conditions for advection tests template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; const Mesh *pmesh = pmb->pmy_mesh; const int ndim = pmesh->ndim; @@ -240,6 +241,7 @@ inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { //! periods. template inline void UserWorkAfterLoop(Mesh *pmesh, ParameterInput *pin, parthenon::SimTime &tm) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; const int nhyd_vars = 5; const int nspec_vars = 4; diff --git a/src/pgen/beam.hpp b/src/pgen/beam.hpp index cf46b6f7..5a1d71fb 100644 --- a/src/pgen/beam.hpp +++ b/src/pgen/beam.hpp @@ -56,6 +56,7 @@ inline void InitBeamParams(MeshBlock *pmb, ParameterInput *pin) { //! \brief Sets initial conditions for BEAM tests template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; const Mesh *pmesh = pmb->pmy_mesh; const int ndim = pmesh->ndim; @@ -121,6 +122,7 @@ inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { //! \brief template inline void BeamInnerX2(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pmb = mbd->GetBlockPointer(); @@ -197,6 +199,7 @@ inline void BeamInnerX2(std::shared_ptr> &mbd, bool coarse) //! \brief template inline void BeamInnerX1(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pmb = mbd->GetBlockPointer(); diff --git a/src/pgen/blast.hpp b/src/pgen/blast.hpp index cc48f6f6..2ee95564 100644 --- a/src/pgen/blast.hpp +++ b/src/pgen/blast.hpp @@ -127,6 +127,7 @@ KOKKOS_INLINE_FUNCTION Real compute_overlap_sph(geometry::BBox bnds, Real rad, //! \brief Sedov blast wave template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; // Extract blast parameters diff --git a/src/pgen/conduction.hpp b/src/pgen/conduction.hpp index c268368f..4b3ca4f9 100644 --- a/src/pgen/conduction.hpp +++ b/src/pgen/conduction.hpp @@ -61,6 +61,7 @@ inline void InitCondParams(MeshBlock *pmb, ParameterInput *pin) { //! \brief template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; // Extract parameters from packages @@ -130,6 +131,7 @@ inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { //! \brief Sets inner X1 boundary condition to the initial condition template void CondBoundaryImpl(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT auto pmb = mbd->GetBlockPointer(); // Artemis package and params @@ -262,6 +264,7 @@ void CondBoundaryImpl(std::shared_ptr> &mbd, bool coarse) { //! \brief template inline void CondBoundary(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT auto pmb = mbd->GetBlockPointer(); auto artemis_pkg = pmb->packages.Get("artemis"); auto &pkg = pmb->packages.Get("gas"); diff --git a/src/pgen/constant.hpp b/src/pgen/constant.hpp index f8424d20..a7181a1a 100644 --- a/src/pgen/constant.hpp +++ b/src/pgen/constant.hpp @@ -52,6 +52,7 @@ static ConstantParams constant_params; //! \brief template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; // Extract parameters from packages diff --git a/src/pgen/disk.hpp b/src/pgen/disk.hpp index 02dcdd6e..b58def40 100644 --- a/src/pgen/disk.hpp +++ b/src/pgen/disk.hpp @@ -381,6 +381,7 @@ DiskICImpl(V1 v, const int b, const int k, const int j, const int i, V2 pco, EOS //! \brief Sets initial conditions for disk problem template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; // Extract artemis package and params @@ -432,6 +433,7 @@ inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { //! \brief Sets inner or outer X1 boundary condition to the initial condition template void DiskBoundaryVisc(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT PARTHENON_REQUIRE(GEOM == Coordinates::cylindrical || GEOM == Coordinates::spherical3D || @@ -633,6 +635,7 @@ void DiskBoundaryVisc(std::shared_ptr> &mbd, bool coarse) { //! \brief Sets inner or outer boundary condition to the initial condition template void DiskBoundaryIC(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT auto pmb = mbd->GetBlockPointer(); auto artemis_pkg = pmb->packages.Get("artemis"); @@ -672,6 +675,7 @@ void DiskBoundaryIC(std::shared_ptr> &mbd, bool coarse) { //! \brief Extrapolation boundary conditions template void DiskBoundaryExtrap(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT const bool lnx = (GEOM != Coordinates::cartesian); auto pmb = mbd->GetBlockPointer(); @@ -889,6 +893,7 @@ void DiskBoundaryExtrap(std::shared_ptr> &mbd, bool coarse) //! \fn AmrTag ProblemCheckRefinementBlock() //! \brief Refinement criterion for disk pgen inline parthenon::AmrTag ProblemCheckRefinementBlock(MeshBlockData *mbd) { + PARTHENON_INSTRUMENT PARTHENON_FAIL("Disk user-defined AMR criterion not yet implemented!"); return AmrTag::same; } diff --git a/src/pgen/gaussian_bump.hpp b/src/pgen/gaussian_bump.hpp index f8ebe069..28fbc180 100644 --- a/src/pgen/gaussian_bump.hpp +++ b/src/pgen/gaussian_bump.hpp @@ -47,6 +47,7 @@ static BumpParams bump_params; //! \brief template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; auto pm = pmb->pmy_mesh; diff --git a/src/pgen/kh.hpp b/src/pgen/kh.hpp index f20c1f47..6a33f4cb 100644 --- a/src/pgen/kh.hpp +++ b/src/pgen/kh.hpp @@ -41,6 +41,7 @@ static KHParams KH_params; //! \brief template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; // Extract parameters from packages diff --git a/src/pgen/linear_wave.hpp b/src/pgen/linear_wave.hpp index 547364de..43cdd269 100644 --- a/src/pgen/linear_wave.hpp +++ b/src/pgen/linear_wave.hpp @@ -114,6 +114,7 @@ KOKKOS_INLINE_FUNCTION void HydroEigensystem(const Real d, const Real v1, const //! \brief Sets initial conditions for linear wave tests template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; const Mesh *pmesh = pmb->pmy_mesh; const int ndim = pmesh->ndim; @@ -270,6 +271,7 @@ inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { //! periods. template inline void UserWorkAfterLoop(Mesh *pmesh, ParameterInput *pin, parthenon::SimTime &tm) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; const int nhydro = 5; const int nvars = nhydro; diff --git a/src/pgen/lw.hpp b/src/pgen/lw.hpp index 07770683..5ed0c7a6 100644 --- a/src/pgen/lw.hpp +++ b/src/pgen/lw.hpp @@ -39,6 +39,7 @@ static LWParams lw_params; //! \brief template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; // Extract parameters from packages diff --git a/src/pgen/pgen.hpp b/src/pgen/pgen.hpp index f7568b88..377e3186 100644 --- a/src/pgen/pgen.hpp +++ b/src/pgen/pgen.hpp @@ -42,6 +42,7 @@ namespace artemis { //! \brief template void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT std::string name = pin->GetString("artemis", "problem"); if (name == "advection") { advection::ProblemGenerator(pmb, pin); @@ -81,6 +82,7 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { //! \brief Driver routine to initialize meshblock data when meshblocks are created template void InitMeshBlockData(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT geometry::InitBlockGeom(pmb, pin); diff --git a/src/pgen/problem_modifier.hpp b/src/pgen/problem_modifier.hpp index 8a8429b7..67f3075d 100644 --- a/src/pgen/problem_modifier.hpp +++ b/src/pgen/problem_modifier.hpp @@ -40,6 +40,7 @@ namespace artemis { //! \brief template void ProblemModifier(parthenon::ParthenonManager *pman) { + PARTHENON_INSTRUMENT using BF = parthenon::BoundaryFace; using ID = parthenon::IndexDomain; diff --git a/src/pgen/rt.hpp b/src/pgen/rt.hpp index 3da91626..42ae7ec5 100644 --- a/src/pgen/rt.hpp +++ b/src/pgen/rt.hpp @@ -40,6 +40,7 @@ static RTParams RT_params; //! \brief template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; // Extract parameters from packages diff --git a/src/pgen/shock.hpp b/src/pgen/shock.hpp index 0d10ef10..9be64caf 100644 --- a/src/pgen/shock.hpp +++ b/src/pgen/shock.hpp @@ -69,6 +69,7 @@ inline void InitShockParams(MeshBlock *pmb, ParameterInput *pin) { //! \brief Sets initial conditions for shock problem template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; // Extract parameters from packages @@ -138,6 +139,7 @@ inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { //! \fn void ProblemGenerator::ShockInnerX1() template inline void ShockInnerX1(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pmb = mbd->GetBlockPointer(); @@ -188,6 +190,7 @@ inline void ShockInnerX1(std::shared_ptr> &mbd, bool coarse) //! \fn void ProblemGenerator::ShockOuterX1() template inline void ShockOuterX1(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pmb = mbd->GetBlockPointer(); diff --git a/src/pgen/strat.hpp b/src/pgen/strat.hpp index 69ca37d2..545f8e8f 100644 --- a/src/pgen/strat.hpp +++ b/src/pgen/strat.hpp @@ -65,6 +65,7 @@ struct StratParams { //! NOTE(PDM): In order for our user-defined BCs to be compatible with restarts, we must //! reset the StratParams struct upon initialization. inline void InitStratParams(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT auto &artemis_pkg = pmb->packages.Get("artemis"); Params ¶ms = artemis_pkg->AllParams(); if (!(params.hasKey("strat_params"))) { @@ -110,6 +111,7 @@ Real InitialDensity(const StratParams &pars, const Real z) { //! \brief Sets initial conditions for shearing box problem template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; // Extract parameters from packages @@ -192,6 +194,7 @@ inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { //! Extrapolation bc + Outflow no inflow template inline void ExtrapInnerX1(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pmb = mbd->GetBlockPointer(); @@ -284,6 +287,7 @@ inline void ExtrapInnerX1(std::shared_ptr> &mbd, bool coarse //! Extrapolation bc + Outflow no inflow template inline void ExtrapOuterX1(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pmb = mbd->GetBlockPointer(); @@ -391,6 +395,7 @@ inline void ExtrapOuterX1(std::shared_ptr> &mbd, bool coarse //! template inline void ShearInnerX2(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pmb = mbd->GetBlockPointer(); @@ -512,6 +517,7 @@ inline void ShearInnerX2(std::shared_ptr> &mbd, bool coarse) //! template inline void ShearOuterX2(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pmb = mbd->GetBlockPointer(); @@ -621,6 +627,7 @@ inline void ShearOuterX2(std::shared_ptr> &mbd, bool coarse) //! Extrapolation bc + Outflow no inflow template inline void ExtrapInnerX3(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pmb = mbd->GetBlockPointer(); @@ -721,6 +728,7 @@ inline void ExtrapInnerX3(std::shared_ptr> &mbd, bool coarse //! \brief Sets BCs on +z boundary in shearing box template inline void ExtrapOuterX3(std::shared_ptr> &mbd, bool coarse) { + PARTHENON_INSTRUMENT // Extrapolation bc + Outflow no inflow using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; diff --git a/src/pgen/thermalization.hpp b/src/pgen/thermalization.hpp index b79c18e8..2079d377 100644 --- a/src/pgen/thermalization.hpp +++ b/src/pgen/thermalization.hpp @@ -34,6 +34,7 @@ namespace thermalization { //! \brief Sets initial conditions for thermal relaxation problem template inline void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; // Extract parameters from packages diff --git a/src/radiation/imc/imc_driver.cpp b/src/radiation/imc/imc_driver.cpp index d833e869..c78a4b01 100644 --- a/src/radiation/imc/imc_driver.cpp +++ b/src/radiation/imc/imc_driver.cpp @@ -29,6 +29,7 @@ namespace IMC { //! \brief Executes thermal IMC transport (Jaybenne) and syncs updated fields template TaskListStatus JaybenneIMC(Mesh *pmesh, const SimTime &tm, const Real dt) { + PARTHENON_INSTRUMENT auto status = Radiation::UpdateRadiationFields(pmesh).Execute(); if (status != TaskListStatus::complete) return status; status = jaybenne::RadiationStep(pmesh, tm, dt).Execute(); diff --git a/src/radiation/moments/matter_coupling.hpp b/src/radiation/moments/matter_coupling.hpp index 3bbb68f3..424067fa 100644 --- a/src/radiation/moments/matter_coupling.hpp +++ b/src/radiation/moments/matter_coupling.hpp @@ -33,6 +33,7 @@ namespace Moments { //! \brief Implementation for simple radiation-matter coupling source template TaskStatus MatterCouplingSimpleImpl(MeshData *u0, const Real dt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using parthenon::variable_names::any; auto pm = u0->GetParentPointer(); @@ -198,6 +199,7 @@ TaskStatus MatterCouplingSimpleImpl(MeshData *u0, const Real dt) { //! \brief Implementation for "full" radiation-matter coupling source template TaskStatus MatterCouplingFullSingleImpl(MeshData *u0, const Real dt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using parthenon::variable_names::any; auto pm = u0->GetParentPointer(); diff --git a/src/radiation/moments/moments.cpp b/src/radiation/moments/moments.cpp index 17b6e050..bb8d1e51 100644 --- a/src/radiation/moments/moments.cpp +++ b/src/radiation/moments/moments.cpp @@ -269,6 +269,7 @@ std::shared_ptr Initialize(ParameterInput *pin, //! \fn TaskStatus Moments::CalculateFluxes //! \brief Evaluates advective fluxes for moments evolution TaskStatus CalculateFluxes(MeshData *md) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; auto &pkg = pm->packages.Get("moments"); @@ -307,6 +308,7 @@ TaskStatus CalculateFluxes(MeshData *md) { //! \fn TaskStatus Moments::FluxSource //! \brief Evaluates coordinate terms from advective fluxes for moments evolution TaskStatus FluxSource(MeshData *md, const Real dt) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; auto &pkg = pm->packages.Get("moments"); @@ -345,6 +347,7 @@ TaskStatus FluxSource(MeshData *md, const Real dt) { //! \brief template TaskStatus MatterCoupling(MeshData *u0, const Real dt) { + PARTHENON_INSTRUMENT auto pm = u0->GetParentPointer(); auto &artemis_pkg = pm->packages.Get("artemis"); diff --git a/src/radiation/moments/moments.hpp b/src/radiation/moments/moments.hpp index 100c11db..443af9fa 100644 --- a/src/radiation/moments/moments.hpp +++ b/src/radiation/moments/moments.hpp @@ -45,6 +45,7 @@ TaskCollection MomentsTasks(Mesh *pmesh, const SimTime &tm, //! \brief Not enrolled in parthenon's determination for global dt template Real EstimateTimeStep(parthenon::Mesh *pmesh) { + PARTHENON_INSTRUMENT auto &moments_pkg = pmesh->packages.Get("moments"); auto ¶ms = moments_pkg->AllParams(); diff --git a/src/radiation/moments/moments_driver.cpp b/src/radiation/moments/moments_driver.cpp index 0fb2676a..35d03d1d 100644 --- a/src/radiation/moments/moments_driver.cpp +++ b/src/radiation/moments/moments_driver.cpp @@ -26,6 +26,7 @@ namespace Moments { template TaskListStatus MomentsDriver(Mesh *pmesh, const SimTime &tm, parthenon::LowStorageIntegrator *integrator) { + PARTHENON_INSTRUMENT // Craft a series of **equal** substeps that sum to the unsplit step const Real dtlimit = Moments::EstimateTimeStep(pmesh); const int nsteps = static_cast(std::ceil(integrator->dt / dtlimit)); @@ -55,6 +56,7 @@ TaskListStatus MomentsDriver(Mesh *pmesh, const SimTime &tm, template TaskCollection MomentsTasks(Mesh *pmesh, const SimTime &tm, parthenon::LowStorageIntegrator *integrator) { + PARTHENON_INSTRUMENT using TQ = TaskQualifier; TaskCollection tc; diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index 84904b27..22bdf9a9 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -81,6 +81,7 @@ Initialize(ParameterInput *pin, ArtemisUtils::Constants &constants, const bool d //! \fn TaskStatus Radiation::SetOpacities //! \brief Routine to set opacitiy fields (when required, e.g., for Jaybenne IMC) TaskStatus SetOpacities(MeshData *md) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -122,6 +123,7 @@ TaskStatus SetOpacities(MeshData *md) { //! \fn TaskCollection Radiation::UpdateRadiationFields //! \brief TaskCollection to set radiation fields (when required, e.g., for Jaybenne IMC) TaskCollection UpdateRadiationFields(Mesh *pmesh) { + PARTHENON_INSTRUMENT TaskCollection tc; TaskID none(0); const int num_partitions = pmesh->DefaultNumPartitions(); diff --git a/src/radiation/raytrace/raytrace.cpp b/src/radiation/raytrace/raytrace.cpp index 076e4298..5459224d 100644 --- a/src/radiation/raytrace/raytrace.cpp +++ b/src/radiation/raytrace/raytrace.cpp @@ -92,6 +92,7 @@ std::shared_ptr Initialize(ParameterInput *pin, //! \fn StateDescriptor RT::MeshResetCommunication //! \brief Reset comm buffers TaskStatus MeshResetCommunication(MeshData *md) { + PARTHENON_INSTRUMENT const int nblocks = md->NumBlocks(); for (int n = 0; n < nblocks; n++) { auto &mbd = md->GetBlockData(n); @@ -106,6 +107,7 @@ TaskStatus MeshResetCommunication(MeshData *md) { //! \fn StateDescriptor RT::MeshSend //! \brief Send boundary comms TaskStatus MeshSend(MeshData *md) { + PARTHENON_INSTRUMENT const int nblocks = md->NumBlocks(); for (int n = 0; n < nblocks; n++) { auto &mbd = md->GetBlockData(n); @@ -120,6 +122,7 @@ TaskStatus MeshSend(MeshData *md) { //! \fn StateDescriptor RT::MeshRecieve //! \brief Recieve comm buffers TaskStatus MeshReceive(MeshData *md) { + PARTHENON_INSTRUMENT TaskStatus status = TaskStatus::complete; const int nblocks = md->NumBlocks(); for (int n = 0; n < nblocks; n++) { @@ -138,6 +141,7 @@ TaskStatus MeshReceive(MeshData *md) { //! \fn StateDescriptor RT::SourceParticles //! \brief Create new particles for the radiation source TaskStatus SourceParticles(MeshData *md, const ParticleWeights &pwght) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &artemis_pkg = pm->packages.Get("artemis"); auto geom = artemis_pkg->Param("coords"); @@ -174,6 +178,7 @@ TaskStatus SourceParticles(MeshData *md, const ParticleWeights &pwght) { //! \fn StateDescriptor RT::PushParticles //! \brief Push the particles through the mesh TaskStatus PushParticles(MeshData *md) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &artemis_pkg = pm->packages.Get("artemis"); @@ -211,6 +216,7 @@ TaskStatus PushParticles(MeshData *md) { //! \fn StateDescriptor RT::RemoveParticles //! \brief Remove particles that have been marked for removal TaskStatus RemoveParticles(MeshData *md) { + PARTHENON_INSTRUMENT for (int b = 0; b < md->NumBlocks(); ++b) { md->GetSwarmData(b)->Get("star")->RemoveMarkedParticles(); @@ -223,6 +229,7 @@ TaskStatus RemoveParticles(MeshData *md) { //! \fn StateDescriptor RT::CheckCompletion //! \brief Determine how many particles are still left to push TaskStatus CheckCompletion(MeshData *md) { + PARTHENON_INSTRUMENT // Taken from jaybenne auto pm = md->GetParentPointer(); // Create SwarmPacks @@ -259,6 +266,7 @@ TaskStatus CheckCompletion(MeshData *md) { //! \fn StateDescriptor RT::EvalOpac //! \brief Evaluate the opacity used for the raytraced radiation TaskStatus EvalOpac(MeshData *md) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -299,6 +307,7 @@ TaskStatus EvalOpac(MeshData *md) { //! \fn StateDescriptor RT::RaytraceDriverTasks //! \brief The driver for the raytrace step TaskCollection RaytraceDriverTasks(Mesh *pmesh, const ParticleWeights &pwght) { + PARTHENON_INSTRUMENT using TQ = TaskQualifier; auto &rt_pkg = pmesh->packages.Get("raytrace"); @@ -342,6 +351,7 @@ TaskCollection RaytraceDriverTasks(Mesh *pmesh, const ParticleWeights &pwght) { //! \fn StateDescriptor RT::RaytraceDriver //! \brief Pepare to call the TaskCollection for raytracing TaskListStatus RaytraceDriver(Mesh *pmesh) { + PARTHENON_INSTRUMENT auto &artemis_pkg = pmesh->packages.Get("artemis"); auto geom = artemis_pkg->Param("coords"); // What is the minimum dtheta, dphi diff --git a/src/radiation/raytrace/raytrace.hpp b/src/radiation/raytrace/raytrace.hpp index 0df668b4..5f4135f0 100644 --- a/src/radiation/raytrace/raytrace.hpp +++ b/src/radiation/raytrace/raytrace.hpp @@ -54,6 +54,7 @@ GetIndices(const parthenon::Coordinates_t &pco, std::array x) { //! \brief Implementation for pushing particles template TaskStatus PushParticlesImpl(MeshData *md, const geometry::CoordParams &cpars) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; auto &rt_pkg = pm->packages.Get("raytrace"); @@ -165,6 +166,7 @@ TaskStatus PushParticlesImpl(MeshData *md, const geometry::CoordParams &cp template TaskStatus SourceParticlesImpl(MeshData *md, const geometry::CoordParams &cpars, const ParticleWeights &pwght) { + PARTHENON_INSTRUMENT // Create SwarmPacks // Create pack diff --git a/src/rotating_frame/advection_driver.cpp b/src/rotating_frame/advection_driver.cpp index 7ff9faba..4d334ef0 100644 --- a/src/rotating_frame/advection_driver.cpp +++ b/src/rotating_frame/advection_driver.cpp @@ -32,6 +32,7 @@ namespace RotatingFrame { //! \fn TaskListStatus RotatingFrame::Advect //! \brief Executes linear advection term for orbital advection TaskListStatus Advect(Mesh *pmesh, const SimTime &tm) { + PARTHENON_INSTRUMENT // Craft a series of **equal** subsetps that sum to the unsplit step const Real dtlimit = EstimateTimestep(pmesh, 1.0); const int nsteps = static_cast(std::ceil(tm.dt / dtlimit)); @@ -58,6 +59,7 @@ TaskListStatus Advect(Mesh *pmesh, const SimTime &tm) { //---------------------------------------------------------------------------------------- //! \fn TaskCollection LinearAdvectionStep TaskCollection LinearAdvectionStep(Mesh *pmesh, const SimTime &tm, const Real scdt) { + PARTHENON_INSTRUMENT TaskCollection tc; if (!(pmesh->ndim >= 2)) return tc; @@ -89,6 +91,7 @@ TaskCollection LinearAdvectionStep(Mesh *pmesh, const SimTime &tm, const Real sc //! \fn TaskStatus RotatingFrame::LagrangeRemap //! \brief TaskStatus LagrangeRemap(MeshData *u0, const Real scdt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; auto pm = u0->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; diff --git a/src/rotating_frame/rotating_frame.cpp b/src/rotating_frame/rotating_frame.cpp index 3a749ec1..07ea9e06 100644 --- a/src/rotating_frame/rotating_frame.cpp +++ b/src/rotating_frame/rotating_frame.cpp @@ -83,6 +83,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { //! \fn TaskStatus RotatingFrame::RotatingFrameForce //! \brief TaskStatus RotatingFrameForce(MeshData *md, const Real time, const Real dt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; auto pm = md->GetParentPointer(); @@ -121,6 +122,7 @@ TaskStatus RotatingFrameForce(MeshData *md, const Real time, const Real dt //! \brief Compute multiple of linear advection timestep (if do_shear) template Real EstimateTimestepMesh(MeshData *md) { + PARTHENON_INSTRUMENT auto pmesh = md->GetParentPointer(); const bool do_shear = pmesh->packages.Get("artemis")->template Param("do_shear"); if (!(do_shear)) return Big(); @@ -134,6 +136,7 @@ Real EstimateTimestepMesh(MeshData *md) { //! \fn Real RotatingFrame::EstimateTimeStep //! \brief Not enrolled in parthenon's determination for global dt Real EstimateTimestep(parthenon::Mesh *pmesh, const Real dt_ratio) { + PARTHENON_INSTRUMENT // Extract rotating frame params auto &rframe_pkg = pmesh->packages.Get("rotating_frame"); const Real &om0 = rframe_pkg->Param("omega"); diff --git a/src/rotating_frame/rotating_frame.hpp b/src/rotating_frame/rotating_frame.hpp index 4681f540..91d4a583 100644 --- a/src/rotating_frame/rotating_frame.hpp +++ b/src/rotating_frame/rotating_frame.hpp @@ -225,6 +225,7 @@ KOKKOS_INLINE_FUNCTION void RemapCons(const geometry::CoordParams &cpars, const template TaskStatus LagrangeRemapImpl(MeshData *u0, const V1 &v0, const V2 &vg, const Real dwdt) { + PARTHENON_INSTRUMENT const int multi_d = u0->GetNDim() >= 2; PARTHENON_REQUIRE(multi_d, "Linear advection does not work in 1D"); const int three_d = u0->GetNDim() == 3; diff --git a/src/rotating_frame/rotating_frame_impl.hpp b/src/rotating_frame/rotating_frame_impl.hpp index 8f65205c..15d0730e 100644 --- a/src/rotating_frame/rotating_frame_impl.hpp +++ b/src/rotating_frame/rotating_frame_impl.hpp @@ -29,6 +29,7 @@ namespace RotatingFrame { //! \brief Calculate the shearing box frame body forces TaskStatus ShearingBoxImpl(MeshData *md, const Real om0, const Real qshear, const bool do_gas, const bool do_dust, const Real dt) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -100,6 +101,7 @@ TaskStatus ShearingBoxImpl(MeshData *md, const Real om0, const Real qshear template TaskStatus RotatingFrameImpl(MeshData *md, const Real om0, const bool do_gas, const bool do_dust, const Real dt) { + PARTHENON_INSTRUMENT // Adds the rotating frame terms to the azimuthal momentum equation and the energy // equation. Note that in comments in this function, R is always the cylindrical radius. diff --git a/src/utils/diffusion/diffusion.hpp b/src/utils/diffusion/diffusion.hpp index 2154d1ba..f5273ec7 100644 --- a/src/utils/diffusion/diffusion.hpp +++ b/src/utils/diffusion/diffusion.hpp @@ -29,6 +29,7 @@ namespace Diffusion { //! \brief Zeroes diffusion fluxes template TaskStatus ZeroDiffusionImpl(MeshData *md, SparsePackFlux vf) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); const auto multi_d = (pm->ndim > 1); const auto three_d = (pm->ndim > 2); @@ -64,6 +65,7 @@ template Real EstimateTimestep(MeshData *md, DiffCoeffParams &dp, PKG &pkg, const EOS &eos, SparsePackPrim vprim) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; auto pm = md->GetParentPointer(); IndexRange ib = md->GetBoundsI(IndexDomain::interior); @@ -120,6 +122,7 @@ template *md, PKG &pkg, SparsePackCons v0, SparsePackPrim p, SparsePackFlux vf, const bool do_viscosity, const Real dt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using TE = parthenon::TopologicalElement; diff --git a/src/utils/diffusion/momentum_diffusion.hpp b/src/utils/diffusion/momentum_diffusion.hpp index 41475798..92b82a75 100644 --- a/src/utils/diffusion/momentum_diffusion.hpp +++ b/src/utils/diffusion/momentum_diffusion.hpp @@ -594,6 +594,7 @@ template TaskStatus MomentumFluxImpl(MeshData *md, DiffCoeffParams dp, PKG &pkg, SparsePackPrim vprim, SparsePackFlux vf) { + PARTHENON_INSTRUMENT PARTHENON_DEBUG_REQUIRE(FLUID_TYPE == Fluid::gas, "Momentum diffusion only works with a gas fluid"); diff --git a/src/utils/diffusion/thermal_diffusion.hpp b/src/utils/diffusion/thermal_diffusion.hpp index daa0155c..12d75014 100644 --- a/src/utils/diffusion/thermal_diffusion.hpp +++ b/src/utils/diffusion/thermal_diffusion.hpp @@ -34,6 +34,7 @@ template TaskStatus ThermalFluxImpl(MeshData *md, DiffCoeffParams dp, PKG &pkg, SparsePackPrim vprim, SparsePackFlux vf) { + PARTHENON_INSTRUMENT // Set heat flux // dE/dt = div(q) // q = - K . grad(T) diff --git a/src/utils/fluxes/fluid_fluxes.hpp b/src/utils/fluxes/fluid_fluxes.hpp index a7ef6754..0b4e3297 100644 --- a/src/utils/fluxes/fluid_fluxes.hpp +++ b/src/utils/fluxes/fluid_fluxes.hpp @@ -75,6 +75,7 @@ template TaskStatus CalculateFluxesImpl(MeshData *md, PKG &pkg, PRIM vp, FLUX vflx, FACE vface, GEO vg) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); // Bounds and indexing @@ -249,6 +250,7 @@ template TaskStatus FluxSourceImpl(MeshData *md, PKG &pkg, PRIM vp, CONS vcons, FACE vface, GEO vg, const Real omf, const Real dt) { + PARTHENON_INSTRUMENT // Indexing and geometry const auto ib = md->GetBoundsI(IndexDomain::interior); const auto jb = md->GetBoundsJ(IndexDomain::interior); diff --git a/src/utils/history.hpp b/src/utils/history.hpp index 761576f0..5bf94332 100644 --- a/src/utils/history.hpp +++ b/src/utils/history.hpp @@ -27,6 +27,7 @@ namespace ArtemisUtils { //! variable specified by VAR template std::vector ReduceSpeciesVolumeIntegral(MeshData *md) { + PARTHENON_INSTRUMENT auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; @@ -68,6 +69,7 @@ std::vector ReduceSpeciesVolumeIntegral(MeshData *md) { //! variable specified by VAR template std::vector ReduceSpeciesVectorVolumeIntegral(MeshData *md) { + PARTHENON_INSTRUMENT PARTHENON_REQUIRE(DIR > 0 && DIR <= 3, "Direction must be X1DIR, X2DIR, or X3DIR!"); auto pm = md->GetParentPointer(); diff --git a/src/utils/integrators/artemis_integrator.hpp b/src/utils/integrators/artemis_integrator.hpp index b52415cd..c7c9f02c 100644 --- a/src/utils/integrators/artemis_integrator.hpp +++ b/src/utils/integrators/artemis_integrator.hpp @@ -28,6 +28,7 @@ namespace ArtemisUtils { //! \fn TaskStatus ArtemisUtils::DeepCopyConservedData //! \brief inline TaskStatus DeepCopyConservedData(MeshData *to, MeshData *from) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using parthenon::variable_names::any; @@ -56,6 +57,7 @@ inline TaskStatus DeepCopyConservedData(MeshData *to, MeshData *from template TaskStatus ApplyUpdate(MeshData *u0, MeshData *u1, const Real g0, const Real g1, const Real beta_dt) { + PARTHENON_INSTRUMENT using parthenon::MakePackDescriptor; using parthenon::variable_names::any; auto pm = u0->GetParentPointer(); diff --git a/src/utils/refinement/amr_criteria.hpp b/src/utils/refinement/amr_criteria.hpp index 977b0e08..bf7548d1 100644 --- a/src/utils/refinement/amr_criteria.hpp +++ b/src/utils/refinement/amr_criteria.hpp @@ -27,6 +27,7 @@ namespace ArtemisUtils { //! \brief template AmrTag ScalarFirstDerivative(MeshBlockData *md) { + PARTHENON_INSTRUMENT auto pmb = md->GetBlockPointer(); auto pm = pmb->pmy_mesh; auto &pco = pmb->coords; @@ -121,6 +122,7 @@ AmrTag ScalarFirstDerivative(MeshBlockData *md) { //! \brief template AmrTag ScalarMagnitude(MeshBlockData *md) { + PARTHENON_INSTRUMENT auto pmb = md->GetBlockPointer(); auto pm = pmb->pmy_mesh; auto &resolved_pkgs = pm->resolved_packages;