Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(checkpoint): refactor checkpoint file naming logic #783

Merged
merged 3 commits into from
Jan 22, 2025
Merged

refactor(checkpoint): refactor checkpoint file naming logic #783

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ea72e12
refactor(checkpoint): refactor checkpoint file naming logic
xyyy1420 Jan 15, 2025
f56f89d
fix
xyyy1420 Jan 15, 2025
3770e52
fix naming
xyyy1420 Jan 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 41 additions & 26 deletions src/checkpoint/serializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,23 +96,25 @@ void Serializer::serializePMem(uint64_t inst_count) {
Log("Put gcpt restorer %s to start of pmem", restorer);
}

string filepath;
string base_file_path;
string memory_file_path;

if (checkpoint_state == SimpointCheckpointing) {
filepath = pathManager.getOutputPath() + "_" + to_string(simpoint2Weights.begin()->first) + "_" +
base_file_path = pathManager.getOutputPath() + "_" + to_string(simpoint2Weights.begin()->first) + "_" +
to_string(simpoint2Weights.begin()->second);
} else {
filepath = pathManager.getOutputPath() + "_" + to_string(inst_count);
base_file_path = pathManager.getOutputPath() + "_" + to_string(inst_count);
}

if (compress_file_format == GZ_FORMAT) {
filepath += "_.gz";
gzFile compressed_mem = gzopen(filepath.c_str(), "wb");
if (compressed_mem == nullptr) {
cerr << "Failed to open " << filepath << endl;
Log("Using GZ format generate checkpoint");
memory_file_path = base_file_path + "_memory_.gz";
gzFile memory_compressed_file = gzopen(memory_file_path.c_str(), "wb");
if (memory_compressed_file == nullptr) {
cerr << "Failed to open " << memory_file_path << endl;
xpanic("Can't open physical memory checkpoint file!\n");
} else {
cout << "Opening " << filepath << " as checkpoint output file" << endl;
cout << "Opening " << memory_file_path << " as checkpoint output file" << endl;
}

uint64_t pass_size = 0;
Expand All @@ -122,39 +124,52 @@ void Serializer::serializePMem(uint64_t inst_count) {
? numeric_limits<int>::max()
: ((int64_t)PMEM_SIZE - (int64_t)written);

if (gzwrite(compressed_mem, pmem + written, (uint32_t)pass_size) != (int)pass_size) {
if (gzwrite(memory_compressed_file, pmem + written, (uint32_t)pass_size) != (int)pass_size) {
xpanic("Write failed on physical memory checkpoint file\n");
}
Log("Written 0x%lx bytes\n", pass_size);
}

if (gzclose(compressed_mem)) {
xpanic("Close failed on physical memory checkpoint file\n");
if (gzclose(memory_compressed_file)) {
xpanic("Close failed on physical checkpoint file\n");
}

} else if (compress_file_format == ZSTD_FORMAT) {
filepath += "_.zstd";
Log("Using ZSTD format generate checkpoint");

memory_file_path += base_file_path + "_memory_.zstd";
Log("Opening %s as memory output file", memory_file_path.c_str());

// zstd compress
size_t const compress_buffer_size = ZSTD_compressBound(PMEM_SIZE);
void *const compress_buffer = malloc(compress_buffer_size);
assert(compress_buffer);
size_t memory_size = PMEM_SIZE;
size_t const memory_compress_buffer_size = ZSTD_compressBound(memory_size);
uint8_t *const memory_compress_buffer = (uint8_t*)malloc(memory_compress_buffer_size);
assert(memory_compress_buffer);


size_t const compress_size = ZSTD_compress(compress_buffer, compress_buffer_size, pmem, PMEM_SIZE, 1);
assert(compress_size <= compress_buffer_size && compress_size != 0);

FILE *compress_file = fopen(filepath.c_str(), "wb");
size_t fw_size = fwrite(compress_buffer, 1, compress_size, compress_file);

if (fw_size != (size_t)compress_size) {
free(compress_buffer);
xpanic("file write error: %s : %s \n", filepath.c_str(), strerror(errno));
// compress flash device memory
size_t memory_compress_size = ZSTD_compress(memory_compress_buffer, memory_compress_buffer_size, pmem, PMEM_SIZE, 1);
assert(memory_compress_size <= memory_compress_buffer_size && memory_compress_size != 0);
Log("pmem compress success, compress size %ld", memory_compress_size);

FILE *memory_compress_file = fopen(memory_file_path.c_str(), "wb");
size_t memory_fw_size = fwrite(memory_compress_buffer, 1, memory_compress_size, memory_compress_file);

if (memory_fw_size != memory_compress_size) {
fclose(memory_compress_file);
free(memory_compress_buffer);
xpanic("file write error: %s : %s\n", memory_file_path.c_str(), strerror(errno));
}

if (fclose(compress_file)) {
free(compress_buffer);
xpanic("file close error: %s : %s \n", filepath.c_str(), strerror(errno));
if (fclose(memory_compress_file)) {
free(memory_compress_buffer);
xpanic("file close error: %s : %s \n", base_file_path.c_str(), strerror(errno));
}

free(compress_buffer);
free(memory_compress_buffer);

} else {
xpanic("You need to specify the compress file format using: --checkpoint-format\n");
}
Expand Down
Loading