Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Error handling on failed writes #904

Draft
wants to merge 10 commits into
base: dev
Choose a base branch
from
Draft

Error handling on failed writes #904

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
e0c17ca
Failing to write bulk file is fatal
lkotipal Feb 5, 2024
8c09856
Attempt to clean up failed restart writes
lkotipal Feb 5, 2024
5f3a100
dummy
lkotipal Apr 22, 2024
25c8877
Don't remove restart if we don't have a filename
lkotipal Apr 23, 2024
8d7dead
Write proper file and line
lkotipal Apr 23, 2024
3abd197
More line numbers
lkotipal Apr 23, 2024
b510444
Reduction of restart write success
lkotipal Apr 23, 2024
17423eb
Reduce scope of restart filename
lkotipal Apr 23, 2024
1fc4e4f
Don't assert
lkotipal Apr 23, 2024
ffc00d2
Do sanity check, actually
lkotipal Apr 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions iowrite.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1522,7 +1522,7 @@ bool writeRestart(
const std::string& versionInfo,
const std::string& configInfo,
DataReducer& dataReducer,
const string& name,
string& name,
const uint& fileIndex,
const int& stripe)
{
Expand Down Expand Up @@ -1557,6 +1557,7 @@ bool writeRestart(
fname.width(7);
fname.fill('0');
fname << fileIndex << "." << currentDate << ".vlsv";
name = fname.str();

phiprof::Timer openTimer {"open"};
//Open the file with vlsvWriter:
Expand Down Expand Up @@ -1588,7 +1589,9 @@ bool writeRestart(
MPI_Info_set(MPIinfo, factor, stripeChar);
}

if( vlsvWriter.open( fname.str(), MPI_COMM_WORLD, masterProcessId, MPIinfo ) == false) return false;
if (vlsvWriter.open( fname.str(), MPI_COMM_WORLD, masterProcessId, MPIinfo ) == false) {
return false;
}

if( MPIinfo != MPI_INFO_NULL ) {
MPI_Info_free(&MPIinfo);
Expand Down
2 changes: 1 addition & 1 deletion iowrite.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ bool writeRestart(
const std::string& versionInfo,
const std::string& configInfo,
DataReducer& dataReducer,
const std::string& name,
std::string& name,
const uint& fileIndex,
const int& stripe
);
Expand Down
42 changes: 21 additions & 21 deletions vlasiator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -590,7 +590,8 @@ int main(int argn,char* args[]) {
writeGhosts
) == false
) {
cerr << "FAILED TO WRITE GRID AT " << __FILE__ << " " << __LINE__ << endl;
// TODO make this std::format when we get C++20
abort_mpi(std::string(__FILE__) + ":" + std::to_string(__LINE__) + ": FAILED TO WRITE GRID", 1);
}

phiprof::stop("Initialization");
Expand Down Expand Up @@ -729,7 +730,8 @@ int main(int argn,char* args[]) {
writeGhosts
) == false
) {
cerr << "FAILED TO WRITE GRID AT " << __FILE__ << " " << __LINE__ << endl;
// TODO make this std::format when we get C++20
abort_mpi(std::string(__FILE__) + ":" + std::to_string(__LINE__) + ": FAILED TO WRITE GRID", 1);
}

P::systemWriteDistributionWriteStride.pop_back();
Expand Down Expand Up @@ -946,7 +948,8 @@ int main(int argn,char* args[]) {
writeGhosts
) == false
) {
cerr << "FAILED TO WRITE GRID AT" << __FILE__ << " " << __LINE__ << endl;
// TODO make this std::format when we get C++20
abort_mpi(std::string(__FILE__) + ":" + std::to_string(__LINE__) + ": FAILED TO WRITE GRID", 1);
}
P::systemWrites[i]++;
// Special case for large timesteps
Expand Down Expand Up @@ -1015,25 +1018,22 @@ int main(int argn,char* args[]) {
if (myRank == MASTER_RANK)
logFile << "(IO): Writing restart data to disk, tstep = " << P::tstep << " t = " << P::t << endl << writeVerbose;
//Write the restart:
if( writeRestart(mpiGrid,
perBGrid, // TODO: Merge all the fsgrids passed here into one meta-object
EGrid,
EHallGrid,
EGradPeGrid,
momentsGrid,
dPerBGrid,
dMomentsGrid,
BgBGrid,
volGrid,
technicalGrid,
version,
config,
outputReducer,"restart",(uint)P::t,P::restartStripeFactor) == false ) {
logFile << "(IO): ERROR Failed to write restart!" << endl << writeVerbose;
cerr << "FAILED TO WRITE RESTART" << endl;
}
// TODO: Merge all the fsgrids passed here into one meta-object
std::string restartFilename {"restart"};
bool restartSuccess {writeRestart(mpiGrid, perBGrid, EGrid, EHallGrid, EGradPeGrid, momentsGrid, dPerBGrid, dMomentsGrid, BgBGrid, volGrid, technicalGrid, version, config, outputReducer, restartFilename, (uint)P::t,P::restartStripeFactor)};
MPI_Reduce(myRank == MASTER_RANK ? MPI_IN_PLACE : &restartSuccess, &restartSuccess, 1, MPI_CXX_BOOL, MPI_LAND, MASTER_RANK, MPI_COMM_WORLD);
if (myRank == MASTER_RANK) {
logFile << "(IO): .... done!"<< endl << writeVerbose;
if(!restartSuccess) {
// If restart write fails, remove the malformed file and hope a human clears space soon
// Sanity check, this should be set before writeRestart returns
if (restartFilename != "restart") {
std::remove(restartFilename.c_str());
}
logFile << "(IO): ERROR Failed to write restart!" << endl << writeVerbose;
cerr << "FAILED TO WRITE RESTART" << endl;
} else {
logFile << "(IO): .... done!"<< endl << writeVerbose;
}
}
timer.stop();
}
Expand Down
Loading