Skip to content

Number of DOF is std::uint64_t #951

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
andrsd wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Number of DOF is std::uint64_t #951

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a1e6ea5
`GetNumLocalNodes` returns `std::uint64_t`
andrsd Dec 19, 2025
bf9a816
`GetNumGlobalDOFs` returns `std::uint64_t`
andrsd Dec 19, 2025
01ae17e
Global vector and matrix size in PETSc wrappers takes `PetscInt`
andrsd Dec 19, 2025
65d9dc8
`GetNumLocalDOFs` returns `std::uint64_t`
andrsd Dec 19, 2025
e20762f
`GetNumGhostDOFs` returns `std::uint64_t`
andrsd Dec 19, 2025
0c8fc0e
`GetNumLocalAndGhostDOFs` returns `std::uint64_t`
andrsd Dec 19, 2025
c5af0ea
`GetNumGlobalNodes` returns `std::uint64_t`
andrsd Dec 19, 2025
dcc7400
`GetNumPhiIterativeUnknowns` returns `std::uint64_t`s
andrsd Dec 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 14 additions & 20 deletions framework/math/petsc_utils/petsc_utils.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,32 +43,30 @@ ThrowPETScError(int ierr, const char* expr, const char* file, int line)
}

Vec
CreateVector(int64_t local_size, int64_t global_size)
CreateVector(PetscInt local_size, PetscInt global_size)
{
Vec x = nullptr;
OpenSnPETScCall(VecCreate(opensn::mpi_comm, &x));
OpenSnPETScCall(VecSetType(x, VECMPI));
OpenSnPETScCall(
VecSetSizes(x, ToPetscInt(local_size, "local_size"), ToPetscInt(global_size, "global_size")));
OpenSnPETScCall(VecSetSizes(x, local_size, global_size));
OpenSnPETScCall(VecSetOption(x, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE));

return x;
}

void
CreateVector(Vec& x, int64_t local_size, int64_t global_size)
CreateVector(Vec& x, PetscInt local_size, PetscInt global_size)
{
OpenSnPETScCall(VecCreate(opensn::mpi_comm, &x));
OpenSnPETScCall(VecSetType(x, VECMPI));
OpenSnPETScCall(
VecSetSizes(x, ToPetscInt(local_size, "local_size"), ToPetscInt(global_size, "global_size")));
OpenSnPETScCall(VecSetSizes(x, local_size, global_size));
OpenSnPETScCall(VecSetOption(x, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE));
}

Vec
CreateVectorWithGhosts(int64_t local_size,
int64_t global_size,
int64_t nghosts,
CreateVectorWithGhosts(PetscInt local_size,
PetscInt global_size,
PetscInt nghosts,
const std::vector<int64_t>& ghost_indices)
{
std::vector<PetscInt> ghost_indices_petsc;
Expand All @@ -79,9 +77,9 @@ CreateVectorWithGhosts(int64_t local_size,
Vec x = nullptr;
OpenSnPETScCall(
VecCreateGhost(opensn::mpi_comm,
ToPetscInt(local_size, "local_size"),
ToPetscInt(global_size, "global_size"),
ToPetscInt(nghosts, "nghosts"),
local_size,
global_size,
nghosts,
(ghost_indices_petsc.empty()) ? nullptr : ghost_indices_petsc.data(),
&x));

Expand All @@ -91,14 +89,12 @@ CreateVectorWithGhosts(int64_t local_size,
}

Mat
CreateSquareMatrix(int64_t local_size, int64_t global_size)
CreateSquareMatrix(PetscInt local_size, PetscInt global_size)
{
Mat A = nullptr;
OpenSnPETScCall(MatCreate(opensn::mpi_comm, &A));
OpenSnPETScCall(MatSetType(A, MATMPIAIJ));
const auto local = ToPetscInt(local_size, "local_size");
const auto global = ToPetscInt(global_size, "global_size");
OpenSnPETScCall(MatSetSizes(A, local, local, global, global));
OpenSnPETScCall(MatSetSizes(A, local_size, local_size, global_size, global_size));

OpenSnPETScCall(MatMPIAIJSetPreallocation(A, 1, nullptr, 0, nullptr));
OpenSnPETScCall(MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE));
Expand All @@ -108,13 +104,11 @@ CreateSquareMatrix(int64_t local_size, int64_t global_size)
}

void
CreateSquareMatrix(Mat& A, int64_t local_size, int64_t global_size)
CreateSquareMatrix(Mat& A, PetscInt local_size, PetscInt global_size)
{
OpenSnPETScCall(MatCreate(opensn::mpi_comm, &A));
OpenSnPETScCall(MatSetType(A, MATMPIAIJ));
const auto local = ToPetscInt(local_size, "local_size");
const auto global = ToPetscInt(global_size, "global_size");
OpenSnPETScCall(MatSetSizes(A, local, local, global, global));
OpenSnPETScCall(MatSetSizes(A, local_size, local_size, global_size, global_size));

OpenSnPETScCall(MatMPIAIJSetPreallocation(A, 1, nullptr, 0, nullptr));
OpenSnPETScCall(MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE));
Expand Down
14 changes: 7 additions & 7 deletions framework/math/petsc_utils/petsc_utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ struct PETScSolverSetup
* return x;
* \endcode
*/
Vec CreateVector(int64_t local_size, int64_t global_size);
Vec CreateVector(PetscInt local_size, PetscInt global_size);

/**
* Creates a general vector.
Expand All @@ -53,7 +53,7 @@ Vec CreateVector(int64_t local_size, int64_t global_size);
* VecSetOption(x,VEC_IGNORE_NEGATIVE_INDICES,PETSC_TRUE);
* \endcode
*/
void CreateVector(Vec& x, int64_t local_size, int64_t global_size);
void CreateVector(Vec& x, PetscInt local_size, PetscInt global_size);

/**
* Creates a general vector with ghost value support.
Expand All @@ -73,9 +73,9 @@ void CreateVector(Vec& x, int64_t local_size, int64_t global_size);
* return x;
* \endcode
*/
Vec CreateVectorWithGhosts(int64_t local_size,
int64_t global_size,
int64_t nghosts,
Vec CreateVectorWithGhosts(PetscInt local_size,
PetscInt global_size,
PetscInt nghosts,
const std::vector<int64_t>& ghost_indices);

/**
Expand All @@ -97,7 +97,7 @@ Vec CreateVectorWithGhosts(int64_t local_size,
* return A;
* \endcode
*/
Mat CreateSquareMatrix(int64_t local_size, int64_t global_size);
Mat CreateSquareMatrix(PetscInt local_size, PetscInt global_size);

/**
* Creates a general square matrix.
Expand All @@ -115,7 +115,7 @@ Mat CreateSquareMatrix(int64_t local_size, int64_t global_size);
* MatSetOption(A, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE);
* \endcode
*/
void CreateSquareMatrix(Mat& A, int64_t local_size, int64_t global_size);
void CreateSquareMatrix(Mat& A, PetscInt local_size, PetscInt global_size);

/**
* Initializes the sparsity pattern of a matrix.
Expand Down
4 changes: 2 additions & 2 deletions ...ath/spatial_discretization/finite_element/piecewise_linear/piecewise_linear_continuous.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,7 @@ PieceWiseLinearContinuous::MapDOFLocal(const Cell& cell,
} // if is_local
else
{
const size_t num_local_dofs = GetNumLocalDOFs(unknown_manager);
const auto num_local_dofs = GetNumLocalDOFs(unknown_manager);
int64_t ghost_local_node_id = -1;
int64_t counter = 0;
for (const auto& vid_gnid : ghost_node_mapping_)
Expand All @@ -508,7 +508,7 @@ PieceWiseLinearContinuous::MapDOFLocal(const Cell& cell,
return address;
}

size_t
std::uint64_t
PieceWiseLinearContinuous::GetNumGhostDOFs(const UnknownManager& unknown_manager) const
{
unsigned int N = unknown_manager.GetTotalUnknownStructureSize();
Expand Down
2 changes: 1 addition & 1 deletion ...math/spatial_discretization/finite_element/piecewise_linear/piecewise_linear_continuous.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class PieceWiseLinearContinuous : public PieceWiseLinearBase
return MapDOFLocal(cell, node, UNITARY_UNKNOWN_MANAGER, 0, 0);
}

size_t GetNumGhostDOFs(const UnknownManager& unknown_manager) const override;
std::uint64_t GetNumGhostDOFs(const UnknownManager& unknown_manager) const override;
Copy link
Copy Markdown
Contributor

@ragusa ragusa Feb 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What will happen if a user, for whatever reason, does not compile with Petsc64? This line hardcodes that PetscInt is expected to be a uint64, right?

Copy link
Copy Markdown
Collaborator

@wdhawkins wdhawkins Mar 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cmake will fail to configure the project if PETSc isn't compiled with 64-bit integers. We specifically check for that and throw an error.

Copy link
Copy Markdown
Collaborator Author

@andrsd andrsd Mar 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. Currently, we stop if trying to build against PETSc with 32-bit indexing, see code.

If 32-bit indexing is desired, then we have much larger problems to fix... Things like sparsity patterns, parallel STL vector, etc...

Also, size_t is 64-bit integer on 64-bit architectures, so this particular change just unifies the used type...


std::vector<uint64_t> GetGhostDOFIndices(const UnknownManager& unknown_manager) const override;

Expand Down
2 changes: 1 addition & 1 deletion .../spatial_discretization/finite_element/piecewise_linear/piecewise_linear_discontinuous.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,7 @@ PieceWiseLinearDiscontinuous::MapDOFLocal(const Cell& cell,
return -1;
}

size_t
std::uint64_t
PieceWiseLinearDiscontinuous::GetNumGhostDOFs(const UnknownManager& unknown_manager) const
{
return 0;
Expand Down
2 changes: 1 addition & 1 deletion ...h/spatial_discretization/finite_element/piecewise_linear/piecewise_linear_discontinuous.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class PieceWiseLinearDiscontinuous : public PieceWiseLinearBase
return MapDOFLocal(cell, node, UNITARY_UNKNOWN_MANAGER, 0, 0);
}

size_t GetNumGhostDOFs(const UnknownManager& unknown_manager) const override;
std::uint64_t GetNumGhostDOFs(const UnknownManager& unknown_manager) const override;

std::vector<uint64_t> GetGhostDOFIndices(const UnknownManager& unknown_manager) const override;

Expand Down
14 changes: 7 additions & 7 deletions framework/math/spatial_discretization/spatial_discretization.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,35 +26,35 @@ SpatialDiscretization::GetGrid() const
return grid_;
}

size_t
std::uint64_t
SpatialDiscretization::GetNumLocalNodes() const
{
return local_base_block_size_;
}

size_t
std::uint64_t
SpatialDiscretization::GetNumLocalDOFs(const UnknownManager& unknown_manager) const
{
unsigned int N = unknown_manager.GetTotalUnknownStructureSize();

return local_base_block_size_ * N;
}

size_t
std::uint64_t
SpatialDiscretization::GetNumGlobalNodes() const
{
return global_base_block_size_;
}

size_t
std::uint64_t
SpatialDiscretization::GetNumGlobalDOFs(const UnknownManager& unknown_manager) const
{
unsigned int N = unknown_manager.GetTotalUnknownStructureSize();

return global_base_block_size_ * N;
}

size_t
std::uint64_t
SpatialDiscretization::GetNumLocalAndGhostDOFs(const UnknownManager& unknown_manager) const
{
return GetNumLocalDOFs(unknown_manager) + GetNumGhostDOFs(unknown_manager);
Expand Down Expand Up @@ -207,7 +207,7 @@ SpatialDiscretization::LocalizePETScVector(Vec petsc_vector,
std::vector<double>& local_vector,
const UnknownManager& unknown_manager) const
{
size_t num_local_dofs = GetNumLocalDOFs(unknown_manager);
auto num_local_dofs = GetNumLocalDOFs(unknown_manager);

CopyVecToSTLvector(petsc_vector, local_vector, num_local_dofs);
}
Expand All @@ -217,7 +217,7 @@ SpatialDiscretization::LocalizePETScVectorWithGhosts(Vec petsc_vector,
std::vector<double>& local_vector,
const UnknownManager& unknown_manager) const
{
size_t num_local_dofs = GetNumLocalAndGhostDOFs(unknown_manager);
auto num_local_dofs = GetNumLocalAndGhostDOFs(unknown_manager);

CopyVecToSTLvectorWithGhosts(petsc_vector, local_vector, num_local_dofs);
}
Expand Down
12 changes: 6 additions & 6 deletions framework/math/spatial_discretization/spatial_discretization.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,28 +81,28 @@ class SpatialDiscretization
virtual uint64_t MapDOFLocal(const Cell& cell, unsigned int node) const = 0;

/// Returns the number of local nodes used in this discretization.
size_t GetNumLocalNodes() const;
std::uint64_t GetNumLocalNodes() const;

/// Returns the number of global nodes used in this discretization.
size_t GetNumGlobalNodes() const;
std::uint64_t GetNumGlobalNodes() const;

/**
* For the unknown structure in the unknown manager, returns the number of local
* degrees-of-freedom.
*/
size_t GetNumLocalDOFs(const UnknownManager& unknown_manager) const;
std::uint64_t GetNumLocalDOFs(const UnknownManager& unknown_manager) const;

/**
* For the unknown structure in the unknown manager, returns the number of global
* degrees-of-freedom.
*/
size_t GetNumGlobalDOFs(const UnknownManager& unknown_manager) const;
std::uint64_t GetNumGlobalDOFs(const UnknownManager& unknown_manager) const;

/**
* For the unknown structure in the unknown manager, returns the number of ghost
* degrees-of-freedom.
*/
virtual size_t GetNumGhostDOFs(const UnknownManager& unknown_manager) const = 0;
virtual std::uint64_t GetNumGhostDOFs(const UnknownManager& unknown_manager) const = 0;

/**
*For the unknown structure in the unknown manager, returns the global IDs of all the ghost
Expand All @@ -114,7 +114,7 @@ class SpatialDiscretization
* For the unknown structure in the unknown manager, returns the number of local- and ghost
* degrees-of-freedom.
*/
size_t GetNumLocalAndGhostDOFs(const UnknownManager& unknown_manager) const;
std::uint64_t GetNumLocalAndGhostDOFs(const UnknownManager& unknown_manager) const;

/**
* For the given cell, returns the number of relevant nodes. The same can be achieved by
Expand Down
6 changes: 3 additions & 3 deletions modules/diffusion/diffusion.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ DiffusionSolver::GetSpatialDiscretization() const
return sdm_;
}

std::pair<size_t, size_t>
std::pair<std::uint64_t, std::uint64_t>
DiffusionSolver::GetNumPhiIterativeUnknowns()
{
return {sdm_.GetNumLocalDOFs(uk_man_), sdm_.GetNumGlobalDOFs(uk_man_)};
Expand All @@ -84,7 +84,7 @@ DiffusionSolver::AddToRHS(const std::vector<double>& values)

PetscScalar* rhs_ptr = nullptr;
OpenSnPETScCall(VecGetArray(rhs_, &rhs_ptr));
for (size_t i = 0; i < num_local_dofs; ++i)
for (std::uint64_t i = 0; i < num_local_dofs; ++i)
rhs_ptr[i] += values[i];
OpenSnPETScCall(VecRestoreArray(rhs_, &rhs_ptr));
}
Expand Down Expand Up @@ -137,7 +137,7 @@ DiffusionSolver::Initialize()
std::vector<int64_t> ghids(ghost_ids.begin(), ghost_ids.end());
rhs_ = CreateVectorWithGhosts(num_local_dofs_,
num_global_dofs_,
static_cast<int64_t>(sdm_.GetNumGhostDOFs(uk_man_)),
static_cast<PetscInt>(sdm_.GetNumGhostDOFs(uk_man_)),
ghids);
}

Expand Down
2 changes: 1 addition & 1 deletion modules/diffusion/diffusion.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ class DiffusionSolver
/// Returns the associated spatial discretization.
const class SpatialDiscretization& GetSpatialDiscretization() const;

std::pair<size_t, size_t> GetNumPhiIterativeUnknowns();
std::pair<std::uint64_t, std::uint64_t> GetNumPhiIterativeUnknowns();

virtual ~DiffusionSolver();

Expand Down
4 changes: 2 additions & 2 deletions modules/diffusion/diffusion_pwlc_solver.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ DiffusionPWLCSolver::DiffusionPWLCSolver(std::string name,
void
DiffusionPWLCSolver::AssembleAand_b(const std::vector<double>& q_vector)
{
const size_t num_local_dofs = sdm_.GetNumLocalAndGhostDOFs(uk_man_);
const auto num_local_dofs = sdm_.GetNumLocalAndGhostDOFs(uk_man_);
OpenSnInvalidArgumentIf(q_vector.size() != num_local_dofs,
std::string("q_vector size mismatch. ") +
std::to_string(q_vector.size()) + " vs " +
Expand Down Expand Up @@ -246,7 +246,7 @@ DiffusionPWLCSolver::AssembleAand_b(const std::vector<double>& q_vector)
void
DiffusionPWLCSolver::Assemble_b(const std::vector<double>& q_vector)
{
const size_t num_local_dofs = sdm_.GetNumLocalAndGhostDOFs(uk_man_);
const auto num_local_dofs = sdm_.GetNumLocalAndGhostDOFs(uk_man_);
OpenSnInvalidArgumentIf(q_vector.size() != num_local_dofs,
std::string("q_vector size mismatch. ") +
std::to_string(q_vector.size()) + " vs " +
Expand Down
10 changes: 5 additions & 5 deletions ...solvers/discrete_ordinates_problem/acceleration/discrete_ordinates_keigen_acceleration.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ DiscreteOrdinatesKEigenAcceleration::NodallyAveragedPWLDVector(const std::vector

const size_t num_unknowns = uk_man.unknowns.size();

const size_t num_cfem_local_dofs = pwlc_sdm.GetNumLocalAndGhostDOFs(uk_man);
const auto num_cfem_local_dofs = pwlc_sdm.GetNumLocalAndGhostDOFs(uk_man);

std::vector<double> cont_input(num_cfem_local_dofs, 0.0);
std::vector<double> cont_input_ctr(num_cfem_local_dofs, 0.0);
Expand Down Expand Up @@ -257,8 +257,8 @@ DiscreteOrdinatesKEigenAcceleration::CopyOnlyPhi0(const std::vector<double>& phi
const auto& phi_uk_man = do_problem_.GetUnknownManager();
const auto gsi = front_gs_.first_group;
const auto gss = front_gs_.GetNumGroups();
const size_t diff_num_local_dofs = pwlc_ptr_ ? diff_sdm.GetNumLocalAndGhostDOFs(diff_uk_man)
: diff_sdm.GetNumLocalDOFs(diff_uk_man);
const auto diff_num_local_dofs = pwlc_ptr_ ? diff_sdm.GetNumLocalAndGhostDOFs(diff_uk_man)
: diff_sdm.GetNumLocalDOFs(diff_uk_man);

if (pwlc_ptr_)
NodallyAveragedPWLDVector(phi_in, copy_only_phi0_tmp_);
Expand Down Expand Up @@ -296,8 +296,8 @@ DiscreteOrdinatesKEigenAcceleration::ProjectBackPhi0(const std::vector<double>&
const auto& phi_uk_man = do_problem_.GetUnknownManager();
const auto gsi = front_gs_.first_group;
const auto gss = front_gs_.GetNumGroups();
const size_t diff_num_local_dofs = pwlc_ptr_ ? diff_sdm.GetNumLocalAndGhostDOFs(diff_uk_man)
: diff_sdm.GetNumLocalDOFs(diff_uk_man);
const auto diff_num_local_dofs = pwlc_ptr_ ? diff_sdm.GetNumLocalAndGhostDOFs(diff_uk_man)
: diff_sdm.GetNumLocalDOFs(diff_uk_man);

OpenSnLogicalErrorIf(input.size() != diff_num_local_dofs, "Input vector size mismatch");

Expand Down
2 changes: 1 addition & 1 deletion .../linear_boltzmann_solvers/discrete_ordinates_problem/acceleration/nl_keigen_acc_context.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ struct NLKEigenDiffContext : public NonLinearSolverContext
int verbosity_level;
KResidualFunctionContext kresid_func_context;

size_t diff_num_local_dofs;
std::uint64_t diff_num_local_dofs;

std::vector<double> phi_l;
std::vector<double> phi_lph_i;
Expand Down
5 changes: 1 addition & 4 deletions .../linear_boltzmann_solvers/discrete_ordinates_problem/acceleration/nl_keigen_acc_solver.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,7 @@ NLKEigenDiffSolver::SetSystemSize()
auto nl_context_ptr = GetNLKDiffContextPtr(context_ptr_, __PRETTY_FUNCTION__);

auto& diff_solver = nl_context_ptr->diff_solver;
auto sizes = diff_solver.GetNumPhiIterativeUnknowns();

num_local_dofs_ = static_cast<PetscInt>(sizes.first);
num_global_dofs_ = static_cast<PetscInt>(sizes.second);
std::tie(num_local_dofs_, num_global_dofs_) = diff_solver.GetNumPhiIterativeUnknowns();
}

void
Expand Down
Loading
Loading