Skip to content

Commit

Permalink
[KAB] cleaned up the code in HDF5LIBS_TestRecoverFile.cpp
Browse files Browse the repository at this point in the history
  • Loading branch information
np04daq committed Nov 19, 2024
1 parent 61e447a commit f4c4769
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 76 deletions.
15 changes: 0 additions & 15 deletions include/hdf5libs/HDF5RawDataFile.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,8 +180,6 @@ class HDF5RawDataFile
template<typename T>
T get_attribute(const std::string& name);
template<typename T>
T get_attribute(const std::string& name, const T& default_value);
template<typename T>
T get_attribute(const HighFive::Group& grp, const std::string& name);
template<typename T>
T get_attribute(const HighFive::DataSet& dset, std::string name);
Expand Down Expand Up @@ -528,19 +526,6 @@ HDF5RawDataFile::get_attribute(const std::string& name)
return value;
}

template<typename T>
T
HDF5RawDataFile::get_attribute(const std::string& name, const T& default_value)
{
if (!m_file_ptr->hasAttribute(name)) {
return default_value;
}
auto attr = m_file_ptr->getAttribute(name);
T value;
attr.read(value);
return value;
}

template<typename T>
T
HDF5RawDataFile::get_attribute(const HighFive::Group& grp, const std::string& name)
Expand Down
169 changes: 108 additions & 61 deletions test/apps/HDF5LIBS_TestRecoverFile.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/**
* @file HDF5RecoverFile.cpp
*
* Preliminary utility to recover a raw data file that did not get closed properly.
* Preliminary utility to recover a raw data file that did not get closed properly.
*
* This is part of the DUNE DAQ Software Suite, copyright 2024.
* Licensing/copyright details are in the COPYING file that you should have
Expand All @@ -12,26 +12,17 @@

#include "daqdataformats/Fragment.hpp"
#include "daqdataformats/SourceID.hpp"
#include "detdataformats/DetID.hpp"
#include "detdataformats/HSIFrame.hpp"
#include "logging/Logging.hpp"
#include "trgdataformats/TriggerObjectOverlay.hpp"

#include <bitset>
#include <fstream>
#include <iostream>
#include <sstream>

#include <string>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>

using namespace dunedaq::hdf5libs;
using namespace dunedaq::daqdataformats;
using namespace dunedaq::detdataformats;
using namespace dunedaq::trgdataformats;

void
print_usage(const char *appname)
print_usage(const char* appname)
{
std::cout << "Usage: " << appname << " [-R] <file_name>" << std::endl;
std::cout << "The default behavior is to simply print out the changes that need to be made." << std::endl;
Expand All @@ -45,20 +36,20 @@ main(int argc, char** argv)
signed char opt;
while ((opt = getopt(argc, argv, "hR")) != -1) {
switch (opt) {
case 'h':
print_usage(argv[0]);
return 1;
case 'R':
do_recovery = true;
break;
default: /* '?' */
print_usage(argv[0]);
return 1;
case 'h':
print_usage(argv[0]);
return 1;
case 'R':
do_recovery = true;
break;
default: /* '?' */
print_usage(argv[0]);
return 1;
}
}

argc -= (optind-1);
argv += (optind-1);
argc -= (optind - 1);
argv += (optind - 1);
if (argc != 2) {
print_usage(argv[0]);
return 1;
Expand All @@ -67,48 +58,62 @@ main(int argc, char** argv)
const std::string ifile_name = std::string(argv[1]);

// open the file for reading, initially
std::unique_ptr< HDF5RawDataFile> h5file_ptr(new HDF5RawDataFile(ifile_name, false));
std::unique_ptr<HDF5RawDataFile> h5file_ptr(new HDF5RawDataFile(ifile_name, false));

// determine a reasonable value for the file-closing time based on the Linux
// file system last-modified time
size_t last_modified_time = 0;
struct stat stat_results;
auto retcode = stat(ifile_name.c_str(), &stat_results);
if (retcode == 0) {last_modified_time = 1000 * stat_results.st_mtime;} // msec

std::string closing_timestamp = h5file_ptr->get_attribute("closing_timestamp", std::string(""));

size_t recorded_size = h5file_ptr->get_attribute("recorded_size", SIZE_MAX);
if (retcode == 0) {
last_modified_time = 1000 * stat_results.st_mtime;
} // msec

// determine a reasonable value for the size of the recorded data in the file by
// looping through all of the records and fragments
size_t calculated_recorded_size = 0;
auto records = h5file_ptr->get_all_record_ids();
for (auto const& record_id : records) {
if (h5file_ptr->is_timeslice_type()) {
try {
auto tsh_ptr = h5file_ptr->get_tsh_ptr(record_id);
calculated_recorded_size += sizeof(*tsh_ptr);
auto tsh_ptr = h5file_ptr->get_tsh_ptr(record_id);
calculated_recorded_size += sizeof(*tsh_ptr);
} catch (std::exception const& excpt) {
// bad record header, we'll skip the whole TimeSlice
continue;
// bad record header, we'll skip the whole TimeSlice
continue;
}
} else {
try {
auto trh_ptr = h5file_ptr->get_trh_ptr(record_id);
calculated_recorded_size += trh_ptr->get_total_size_bytes();
auto trh_ptr = h5file_ptr->get_trh_ptr(record_id);
calculated_recorded_size += trh_ptr->get_total_size_bytes();
} catch (std::exception const& excpt) {
// bad record header, we'll skip the whole TriggerRecord
continue;
// bad record header, we'll skip the whole TriggerRecord
continue;
}
}
std::set<SourceID> frag_sid_list = h5file_ptr->get_fragment_source_ids(record_id);
for (auto const& source_id : frag_sid_list) {
try {
auto frag_ptr = h5file_ptr->get_frag_ptr(record_id, source_id);
calculated_recorded_size += frag_ptr->get_size();
auto frag_ptr = h5file_ptr->get_frag_ptr(record_id, source_id);
calculated_recorded_size += frag_ptr->get_size();
} catch (std::exception const& excpt) {
// nothing to do, just leave this fragment out of the sum
// nothing to do, just leave this fragment out of the sum
}
}
}

std::vector<std::string> attr_names = h5file_ptr->get_attribute_names();
bool ct_attr_exists = (std::count(attr_names.begin(), attr_names.end(), "closing_timestamp") >= 1);
bool rs_attr_exists = (std::count(attr_names.begin(), attr_names.end(), "recorded_size") >= 1);

std::string closing_timestamp = "";
if (ct_attr_exists) {
closing_timestamp = h5file_ptr->get_attribute<std::string>("closing_timestamp");
}
size_t recorded_size = SIZE_MAX;
if (rs_attr_exists) {
recorded_size = h5file_ptr->get_attribute<size_t>("recorded_size");
}

std::cout << std::endl;
std::cout << std::endl;
Expand All @@ -120,44 +125,86 @@ main(int argc, char** argv)
h5file_ptr.reset();
h5file_ptr.reset(new HDF5RawDataFile(ifile_name, true));

std::cout << "Setting the \"recorded_size\" Attribute value to " << calculated_recorded_size << "." << std::endl;
h5file_ptr->write_attribute("recorded_size", calculated_recorded_size);
if (!ct_attr_exists) {
std::string file_closing_timestamp = std::to_string(last_modified_time);
std::cout << "Setting the \"closing_timestamp\" Attribute value to " << file_closing_timestamp << "."
<< std::endl;
h5file_ptr->write_attribute("closing_timestamp", file_closing_timestamp);
} else {
std::cout << "The \"closing_timestamp\" Attribute in the file is currently set to " << closing_timestamp
<< ", and it will not be over-written." << std::endl;
}

std::string file_closing_timestamp = std::to_string(last_modified_time);
std::cout << "Setting the \"closing_timestamp\" Attribute value to " << file_closing_timestamp << "." << std::endl;
h5file_ptr->write_attribute("closing_timestamp", file_closing_timestamp);
if (!rs_attr_exists) {
std::cout << "Setting the \"recorded_size\" Attribute value to " << calculated_recorded_size << "." << std::endl;
h5file_ptr->write_attribute("recorded_size", calculated_recorded_size);
} else {
std::cout << "The \"recorded_size\" Attribute in the file is currently set to " << recorded_size
<< ", and it will not be over-written." << std::endl;
}

int64_t timestamp =
std::chrono::duration_cast<std::chrono::milliseconds>(system_clock::now().time_since_epoch()).count();
std::cout << "Setting the \"file_recovery_timestamp\" Attribute value to " << timestamp << "." << std::endl;
h5file_ptr->write_attribute("file_recovery_timestamp", timestamp);
if (std::count(attr_names.begin(), attr_names.end(), "file_recovery_timestamp") == 0) {
int64_t timestamp =
std::chrono::duration_cast<std::chrono::milliseconds>(system_clock::now().time_since_epoch()).count();
std::cout << "Setting the \"file_recovery_timestamp\" Attribute value to " << timestamp << "." << std::endl;
h5file_ptr->write_attribute("file_recovery_timestamp", timestamp);
} else {
size_t fr_timestamp = h5file_ptr->get_attribute<size_t>("file_recovery_timestamp");
std::cout << "The \"file_recovery_timestamp\" Attribute in the file is currently set to " << fr_timestamp
<< ", and it will not be over-written." << std::endl;
}

// the HDF5RawDataFile Destructor will handle the file renaming, if that is needed.

} else {

if (closing_timestamp == "") {
std::cout << "The \"closing_timestamp\" Attribute is *not* currently set in the file, and it will be " << std::endl;
std::cout << " set to " << last_modified_time << " if/when the file is recovered." << std::endl;
if (std::count(attr_names.begin(), attr_names.end(), "creation_timestamp") == 0) {
std::cout << "The \"creation_timestamp\" Attribute is *not* currently set in the file." << std::endl;
} else {
std::string creation_timestamp = h5file_ptr->get_attribute<std::string>("creation_timestamp");
char* nds = 0;
time_t blah = strtol(creation_timestamp.c_str(), &nds, 10) / 1000;
tm* gmtm = gmtime(&blah);
std::cout << "The \"creation_timestamp\" Attribute in the file is currently set to " << creation_timestamp
<< "," << std::endl << " which corresponds to (UTC) " << asctime(gmtm);
}

if (!ct_attr_exists) {
std::cout << "The \"closing_timestamp\" Attribute is *not* currently set in the file, and it will be "
<< std::endl << " set to " << last_modified_time << " if/when the file is recovered." << std::endl;
} else {
std::cout << "The \"closing_timestamp\" Attribute in the file is currently set to " << closing_timestamp << "." << std::endl;
char* nds = 0;
time_t blah = strtol(closing_timestamp.c_str(), &nds, 10) / 1000;
tm* gmtm = gmtime(&blah);
std::cout << "The \"closing_timestamp\" Attribute in the file is currently set to " << closing_timestamp
<< "," << std::endl << " which corresponds to (UTC) " << asctime(gmtm);
}

if (recorded_size == SIZE_MAX) {
std::cout << "The \"recorded_size\" Attribute is *not* currently set in the file, and it will be " << std::endl;
std::cout << " set to " << calculated_recorded_size << " if/when the file is recovered." << std::endl;
if (!rs_attr_exists) {
std::cout << "The \"recorded_size\" Attribute is *not* currently set in the file, and it will be " << std::endl
<< " set to " << calculated_recorded_size << " if/when the file is recovered." << std::endl;
} else {
std::cout << "The \"recorded_size\" Attribute in the file is currently set to " << recorded_size << "." << std::endl;
std::cout << "The \"recorded_size\" Attribute in the file is currently set to " << recorded_size << "."
<< std::endl;
}

if (std::count(attr_names.begin(), attr_names.end(), "file_recovery_timestamp") == 0) {
std::cout << "The \"file_recovery_timestamp\" Attribute is *not* currently set in the file." << std::endl;
} else {
size_t fr_timestamp = h5file_ptr->get_attribute<size_t>("file_recovery_timestamp");
time_t blah = fr_timestamp / 1000;
tm* gmtm = gmtime(&blah);
std::cout << "The \"file_recovery_timestamp\" Attribute in the file is currently set to " << fr_timestamp
<< "," << std::endl << " which corresponds to (UTC) " << asctime(gmtm);
}

if (ifile_name.rfind(HDF5RawDataFile::s_inprogress_suffix) != std::string::npos) {
std::cout << "The file *does* have the \"" << HDF5RawDataFile::s_inprogress_suffix << "\" suffix, "
<< "so it will be renamed if/when it is recovered." << std::endl;
<< "so it will be renamed if/when it is recovered." << std::endl;
} else {
std::cout << "The file name does not have the \"" << HDF5RawDataFile::s_inprogress_suffix << "\" suffix, "
<< "so it will not be renamed." << std::endl;
<< "so it will not be renamed." << std::endl;
}

}

return 0;
Expand Down

0 comments on commit f4c4769

Please sign in to comment.