diff --git a/include/caffe/util/hdf5.hpp b/include/caffe/util/hdf5.hpp deleted file mode 100644 index ce568c5..0000000 --- a/include/caffe/util/hdf5.hpp +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef CAFFE_UTIL_HDF5_H_ -#define CAFFE_UTIL_HDF5_H_ - -#include - -#include "hdf5.h" -#include "hdf5_hl.h" - -#include "caffe/blob.hpp" - -namespace caffe { - -template -void hdf5_load_nd_dataset_helper( - hid_t file_id, const char* dataset_name_, int min_dim, int max_dim, - Blob* blob); - -template -void hdf5_load_nd_dataset( - hid_t file_id, const char* dataset_name_, int min_dim, int max_dim, - Blob* blob); - -template -void hdf5_save_nd_dataset( - const hid_t file_id, const string& dataset_name, const Blob& blob, - bool write_diff = false); - -int hdf5_load_int(hid_t loc_id, const string& dataset_name); -void hdf5_save_int(hid_t loc_id, const string& dataset_name, int i); -string hdf5_load_string(hid_t loc_id, const string& dataset_name); -void hdf5_save_string(hid_t loc_id, const string& dataset_name, - const string& s); - -int hdf5_get_num_links(hid_t loc_id); -string hdf5_get_name_by_idx(hid_t loc_id, int idx); - -} // namespace caffe - -#endif // CAFFE_UTIL_HDF5_H_ diff --git a/src/caffe/common.cpp b/src/caffe/common.cpp index dee6816..e365930 100644 --- a/src/caffe/common.cpp +++ b/src/caffe/common.cpp @@ -7,6 +7,8 @@ #include "caffe/common.hpp" #include "caffe/util/rng.hpp" +#include + namespace caffe { // Make sure each thread can have different values. @@ -45,8 +47,8 @@ void GlobalInit(int* pargc, char*** pargv) { ::gflags::ParseCommandLineFlags(pargc, pargv, true); // Google logging. ::google::InitGoogleLogging(*(pargv)[0]); - // Provide a backtrace on segfault. - ::google::InstallFailureSignalHandler(); + //// Provide a backtrace on segfault. + //::google::InstallFailureSignalHandler(); } #ifdef CPU_ONLY // CPU-only Caffe. diff --git a/src/caffe/layers/hdf5_data_layer.cpp b/src/caffe/layers/hdf5_data_layer.cpp deleted file mode 100644 index 2f13dc6..0000000 --- a/src/caffe/layers/hdf5_data_layer.cpp +++ /dev/null @@ -1,166 +0,0 @@ -/* -TODO: -- load file in a separate thread ("prefetch") -- can be smarter about the memcpy call instead of doing it row-by-row - :: use util functions caffe_copy, and Blob->offset() - :: don't forget to update hdf5_daa_layer.cu accordingly -- add ability to shuffle filenames if flag is set -*/ -#include // NOLINT(readability/streams) -#include -#include - -#include "hdf5.h" -#include "hdf5_hl.h" -#include "stdint.h" - -#include "caffe/layers/hdf5_data_layer.hpp" -#include "caffe/util/hdf5.hpp" - -namespace caffe { - -template -HDF5DataLayer::~HDF5DataLayer() { } - -// Load data and label from HDF5 filename into the class property blobs. -template -void HDF5DataLayer::LoadHDF5FileData(const char* filename) { - DLOG(INFO) << "Loading HDF5 file: " << filename; - hid_t file_id = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); - if (file_id < 0) { - LOG(FATAL) << "Failed opening HDF5 file: " << filename; - } - - int top_size = this->layer_param_.top_size(); - hdf_blobs_.resize(top_size); - - const int MIN_DATA_DIM = 1; - const int MAX_DATA_DIM = INT_MAX; - - for (int i = 0; i < top_size; ++i) { - hdf_blobs_[i] = shared_ptr >(new Blob()); - hdf5_load_nd_dataset(file_id, this->layer_param_.top(i).c_str(), - MIN_DATA_DIM, MAX_DATA_DIM, hdf_blobs_[i].get()); - } - - herr_t status = H5Fclose(file_id); - CHECK_GE(status, 0) << "Failed to close HDF5 file: " << filename; - - // MinTopBlobs==1 guarantees at least one top blob - CHECK_GE(hdf_blobs_[0]->num_axes(), 1) << "Input must have at least 1 axis."; - const int num = hdf_blobs_[0]->shape(0); - for (int i = 1; i < top_size; ++i) { - CHECK_EQ(hdf_blobs_[i]->shape(0), num); - } - // Default to identity permutation. - data_permutation_.clear(); - data_permutation_.resize(hdf_blobs_[0]->shape(0)); - for (int i = 0; i < hdf_blobs_[0]->shape(0); i++) - data_permutation_[i] = i; - - // Shuffle if needed. - if (this->layer_param_.hdf5_data_param().shuffle()) { - std::random_shuffle(data_permutation_.begin(), data_permutation_.end()); - DLOG(INFO) << "Successully loaded " << hdf_blobs_[0]->shape(0) - << " rows (shuffled)"; - } else { - DLOG(INFO) << "Successully loaded " << hdf_blobs_[0]->shape(0) << " rows"; - } -} - -template -void HDF5DataLayer::LayerSetUp(const vector*>& bottom, - const vector*>& top) { - // Refuse transformation parameters since HDF5 is totally generic. - CHECK(!this->layer_param_.has_transform_param()) << - this->type() << " does not transform data."; - // Read the source to parse the filenames. - const string& source = this->layer_param_.hdf5_data_param().source(); - LOG(INFO) << "Loading list of HDF5 filenames from: " << source; - hdf_filenames_.clear(); - std::ifstream source_file(source.c_str()); - if (source_file.is_open()) { - std::string line; - while (source_file >> line) { - hdf_filenames_.push_back(line); - } - } else { - LOG(FATAL) << "Failed to open source file: " << source; - } - source_file.close(); - num_files_ = hdf_filenames_.size(); - current_file_ = 0; - LOG(INFO) << "Number of HDF5 files: " << num_files_; - CHECK_GE(num_files_, 1) << "Must have at least 1 HDF5 filename listed in " - << source; - - file_permutation_.clear(); - file_permutation_.resize(num_files_); - // Default to identity permutation. - for (int i = 0; i < num_files_; i++) { - file_permutation_[i] = i; - } - - // Shuffle if needed. - if (this->layer_param_.hdf5_data_param().shuffle()) { - std::random_shuffle(file_permutation_.begin(), file_permutation_.end()); - } - - // Load the first HDF5 file and initialize the line counter. - LoadHDF5FileData(hdf_filenames_[file_permutation_[current_file_]].c_str()); - current_row_ = 0; - - // Reshape blobs. - const int batch_size = this->layer_param_.hdf5_data_param().batch_size(); - const int top_size = this->layer_param_.top_size(); - vector top_shape; - for (int i = 0; i < top_size; ++i) { - top_shape.resize(hdf_blobs_[i]->num_axes()); - top_shape[0] = batch_size; - for (int j = 1; j < top_shape.size(); ++j) { - top_shape[j] = hdf_blobs_[i]->shape(j); - } - top[i]->Reshape(top_shape); - } -} - -template -void HDF5DataLayer::Forward_cpu(const vector*>& bottom, - const vector*>& top) { - const int batch_size = this->layer_param_.hdf5_data_param().batch_size(); - for (int i = 0; i < batch_size; ++i, ++current_row_) { - if (current_row_ == hdf_blobs_[0]->shape(0)) { - if (num_files_ > 1) { - ++current_file_; - if (current_file_ == num_files_) { - current_file_ = 0; - if (this->layer_param_.hdf5_data_param().shuffle()) { - std::random_shuffle(file_permutation_.begin(), - file_permutation_.end()); - } - DLOG(INFO) << "Looping around to first file."; - } - LoadHDF5FileData( - hdf_filenames_[file_permutation_[current_file_]].c_str()); - } - current_row_ = 0; - if (this->layer_param_.hdf5_data_param().shuffle()) - std::random_shuffle(data_permutation_.begin(), data_permutation_.end()); - } - for (int j = 0; j < this->layer_param_.top_size(); ++j) { - int data_dim = top[j]->count() / top[j]->shape(0); - caffe_copy(data_dim, - &hdf_blobs_[j]->cpu_data()[data_permutation_[current_row_] - * data_dim], &top[j]->mutable_cpu_data()[i * data_dim]); - } - } -} - -#ifdef CPU_ONLY -STUB_GPU_FORWARD(HDF5DataLayer, Forward); -#endif - -INSTANTIATE_CLASS(HDF5DataLayer); -REGISTER_LAYER_CLASS(HDF5Data); - -} // namespace caffe diff --git a/src/caffe/layers/hdf5_data_layer.cu b/src/caffe/layers/hdf5_data_layer.cu deleted file mode 100644 index 595d223..0000000 --- a/src/caffe/layers/hdf5_data_layer.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* -TODO: -- only load parts of the file, in accordance with a prototxt param "max_mem" -*/ - -#include -#include - -#include "hdf5.h" -#include "hdf5_hl.h" - -#include "caffe/layers/hdf5_data_layer.hpp" - -namespace caffe { - -template -void HDF5DataLayer::Forward_gpu(const vector*>& bottom, - const vector*>& top) { - const int batch_size = this->layer_param_.hdf5_data_param().batch_size(); - for (int i = 0; i < batch_size; ++i, ++current_row_) { - if (current_row_ == hdf_blobs_[0]->shape(0)) { - if (num_files_ > 1) { - current_file_ += 1; - if (current_file_ == num_files_) { - current_file_ = 0; - if (this->layer_param_.hdf5_data_param().shuffle()) { - std::random_shuffle(file_permutation_.begin(), - file_permutation_.end()); - } - DLOG(INFO) << "Looping around to first file."; - } - LoadHDF5FileData( - hdf_filenames_[file_permutation_[current_file_]].c_str()); - } - current_row_ = 0; - if (this->layer_param_.hdf5_data_param().shuffle()) - std::random_shuffle(data_permutation_.begin(), data_permutation_.end()); - } - for (int j = 0; j < this->layer_param_.top_size(); ++j) { - int data_dim = top[j]->count() / top[j]->shape(0); - caffe_copy(data_dim, - &hdf_blobs_[j]->cpu_data()[data_permutation_[current_row_] - * data_dim], &top[j]->mutable_gpu_data()[i * data_dim]); - } - } -} - -INSTANTIATE_LAYER_GPU_FUNCS(HDF5DataLayer); - -} // namespace caffe diff --git a/src/caffe/layers/hdf5_output_layer.cpp b/src/caffe/layers/hdf5_output_layer.cpp deleted file mode 100644 index f8f1edc..0000000 --- a/src/caffe/layers/hdf5_output_layer.cpp +++ /dev/null @@ -1,74 +0,0 @@ -#include - -#include "hdf5.h" -#include "hdf5_hl.h" - -#include "caffe/layers/hdf5_output_layer.hpp" -#include "caffe/util/hdf5.hpp" - -namespace caffe { - -template -void HDF5OutputLayer::LayerSetUp(const vector*>& bottom, - const vector*>& top) { - file_name_ = this->layer_param_.hdf5_output_param().file_name(); - file_id_ = H5Fcreate(file_name_.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, - H5P_DEFAULT); - CHECK_GE(file_id_, 0) << "Failed to open HDF5 file" << file_name_; - file_opened_ = true; -} - -template -HDF5OutputLayer::~HDF5OutputLayer() { - if (file_opened_) { - herr_t status = H5Fclose(file_id_); - CHECK_GE(status, 0) << "Failed to close HDF5 file " << file_name_; - } -} - -template -void HDF5OutputLayer::SaveBlobs() { - // TODO: no limit on the number of blobs - LOG(INFO) << "Saving HDF5 file " << file_name_; - CHECK_EQ(data_blob_.num(), label_blob_.num()) << - "data blob and label blob must have the same batch size"; - hdf5_save_nd_dataset(file_id_, HDF5_DATA_DATASET_NAME, data_blob_); - hdf5_save_nd_dataset(file_id_, HDF5_DATA_LABEL_NAME, label_blob_); - LOG(INFO) << "Successfully saved " << data_blob_.num() << " rows"; -} - -template -void HDF5OutputLayer::Forward_cpu(const vector*>& bottom, - const vector*>& top) { - CHECK_GE(bottom.size(), 2); - CHECK_EQ(bottom[0]->num(), bottom[1]->num()); - data_blob_.Reshape(bottom[0]->num(), bottom[0]->channels(), - bottom[0]->height(), bottom[0]->width()); - label_blob_.Reshape(bottom[1]->num(), bottom[1]->channels(), - bottom[1]->height(), bottom[1]->width()); - const int data_datum_dim = bottom[0]->count() / bottom[0]->num(); - const int label_datum_dim = bottom[1]->count() / bottom[1]->num(); - - for (int i = 0; i < bottom[0]->num(); ++i) { - caffe_copy(data_datum_dim, &bottom[0]->cpu_data()[i * data_datum_dim], - &data_blob_.mutable_cpu_data()[i * data_datum_dim]); - caffe_copy(label_datum_dim, &bottom[1]->cpu_data()[i * label_datum_dim], - &label_blob_.mutable_cpu_data()[i * label_datum_dim]); - } - SaveBlobs(); -} - -template -void HDF5OutputLayer::Backward_cpu(const vector*>& top, - const vector& propagate_down, const vector*>& bottom) { - return; -} - -#ifdef CPU_ONLY -STUB_GPU(HDF5OutputLayer); -#endif - -INSTANTIATE_CLASS(HDF5OutputLayer); -REGISTER_LAYER_CLASS(HDF5Output); - -} // namespace caffe diff --git a/src/caffe/layers/hdf5_output_layer.cu b/src/caffe/layers/hdf5_output_layer.cu deleted file mode 100644 index c1685cd..0000000 --- a/src/caffe/layers/hdf5_output_layer.cu +++ /dev/null @@ -1,39 +0,0 @@ -#include - -#include "hdf5.h" -#include "hdf5_hl.h" - -#include "caffe/layers/hdf5_output_layer.hpp" - -namespace caffe { - -template -void HDF5OutputLayer::Forward_gpu(const vector*>& bottom, - const vector*>& top) { - CHECK_GE(bottom.size(), 2); - CHECK_EQ(bottom[0]->num(), bottom[1]->num()); - data_blob_.Reshape(bottom[0]->num(), bottom[0]->channels(), - bottom[0]->height(), bottom[0]->width()); - label_blob_.Reshape(bottom[1]->num(), bottom[1]->channels(), - bottom[1]->height(), bottom[1]->width()); - const int data_datum_dim = bottom[0]->count() / bottom[0]->num(); - const int label_datum_dim = bottom[1]->count() / bottom[1]->num(); - - for (int i = 0; i < bottom[0]->num(); ++i) { - caffe_copy(data_datum_dim, &bottom[0]->gpu_data()[i * data_datum_dim], - &data_blob_.mutable_cpu_data()[i * data_datum_dim]); - caffe_copy(label_datum_dim, &bottom[1]->gpu_data()[i * label_datum_dim], - &label_blob_.mutable_cpu_data()[i * label_datum_dim]); - } - SaveBlobs(); -} - -template -void HDF5OutputLayer::Backward_gpu(const vector*>& top, - const vector& propagate_down, const vector*>& bottom) { - return; -} - -INSTANTIATE_LAYER_GPU_FUNCS(HDF5OutputLayer); - -} // namespace caffe diff --git a/src/caffe/net.cpp b/src/caffe/net.cpp index f0bf594..59f12c6 100644 --- a/src/caffe/net.cpp +++ b/src/caffe/net.cpp @@ -5,20 +5,15 @@ #include #include -#include "hdf5.h" - #include "caffe/common.hpp" #include "caffe/layer.hpp" #include "caffe/net.hpp" #include "caffe/parallel.hpp" #include "caffe/proto/caffe.pb.h" -#include "caffe/util/hdf5.hpp" #include "caffe/util/insert_splits.hpp" #include "caffe/util/math_functions.hpp" #include "caffe/util/upgrade_proto.hpp" -#include "caffe/test/test_caffe_main.hpp" - namespace caffe { template @@ -795,52 +790,53 @@ void Net::CopyTrainedLayersFromBinaryProto( template void Net::CopyTrainedLayersFromHDF5(const string trained_filename) { - hid_t file_hid = H5Fopen(trained_filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT); - CHECK_GE(file_hid, 0) << "Couldn't open " << trained_filename; - hid_t data_hid = H5Gopen2(file_hid, "data", H5P_DEFAULT); - CHECK_GE(data_hid, 0) << "Error reading weights from " << trained_filename; - int num_layers = hdf5_get_num_links(data_hid); - for (int i = 0; i < num_layers; ++i) { - string source_layer_name = hdf5_get_name_by_idx(data_hid, i); - if (!layer_names_index_.count(source_layer_name)) { - LOG(INFO) << "Ignoring source layer " << source_layer_name; - continue; - } - int target_layer_id = layer_names_index_[source_layer_name]; - DLOG(INFO) << "Copying source layer " << source_layer_name; - vector > >& target_blobs = - layers_[target_layer_id]->blobs(); - hid_t layer_hid = H5Gopen2(data_hid, source_layer_name.c_str(), - H5P_DEFAULT); - CHECK_GE(layer_hid, 0) - << "Error reading weights from " << trained_filename; - // Check that source layer doesn't have more params than target layer - int num_source_params = hdf5_get_num_links(layer_hid); - CHECK_LE(num_source_params, target_blobs.size()) - << "Incompatible number of blobs for layer " << source_layer_name; - for (int j = 0; j < target_blobs.size(); ++j) { - ostringstream oss; - oss << j; - string dataset_name = oss.str(); - int target_net_param_id = param_id_vecs_[target_layer_id][j]; - if (!H5Lexists(layer_hid, dataset_name.c_str(), H5P_DEFAULT)) { - // Target param doesn't exist in source weights... - if (param_owners_[target_net_param_id] != -1) { - // ...but it's weight-shared in target, so that's fine. - continue; - } else { - LOG(FATAL) << "Incompatible number of blobs for layer " - << source_layer_name; - } - } - hdf5_load_nd_dataset(layer_hid, dataset_name.c_str(), 0, kMaxBlobAxes, - target_blobs[j].get()); - } - H5Gclose(layer_hid); - } - H5Gclose(data_hid); - H5Fclose(file_hid); + NOT_IMPLEMENTED; + //hid_t file_hid = H5Fopen(trained_filename.c_str(), H5F_ACC_RDONLY, + // H5P_DEFAULT); + //CHECK_GE(file_hid, 0) << "Couldn't open " << trained_filename; + //hid_t data_hid = H5Gopen2(file_hid, "data", H5P_DEFAULT); + //CHECK_GE(data_hid, 0) << "Error reading weights from " << trained_filename; + //int num_layers = hdf5_get_num_links(data_hid); + //for (int i = 0; i < num_layers; ++i) { + // string source_layer_name = hdf5_get_name_by_idx(data_hid, i); + // if (!layer_names_index_.count(source_layer_name)) { + // LOG(INFO) << "Ignoring source layer " << source_layer_name; + // continue; + // } + // int target_layer_id = layer_names_index_[source_layer_name]; + // DLOG(INFO) << "Copying source layer " << source_layer_name; + // vector > >& target_blobs = + // layers_[target_layer_id]->blobs(); + // hid_t layer_hid = H5Gopen2(data_hid, source_layer_name.c_str(), + // H5P_DEFAULT); + // CHECK_GE(layer_hid, 0) + // << "Error reading weights from " << trained_filename; + // // Check that source layer doesn't have more params than target layer + // int num_source_params = hdf5_get_num_links(layer_hid); + // CHECK_LE(num_source_params, target_blobs.size()) + // << "Incompatible number of blobs for layer " << source_layer_name; + // for (int j = 0; j < target_blobs.size(); ++j) { + // ostringstream oss; + // oss << j; + // string dataset_name = oss.str(); + // int target_net_param_id = param_id_vecs_[target_layer_id][j]; + // if (!H5Lexists(layer_hid, dataset_name.c_str(), H5P_DEFAULT)) { + // // Target param doesn't exist in source weights... + // if (param_owners_[target_net_param_id] != -1) { + // // ...but it's weight-shared in target, so that's fine. + // continue; + // } else { + // LOG(FATAL) << "Incompatible number of blobs for layer " + // << source_layer_name; + // } + // } + // hdf5_load_nd_dataset(layer_hid, dataset_name.c_str(), 0, kMaxBlobAxes, + // target_blobs[j].get()); + // } + // H5Gclose(layer_hid); + //} + //H5Gclose(data_hid); + //H5Fclose(file_hid); } template @@ -857,59 +853,60 @@ void Net::ToProto(NetParameter* param, bool write_diff) const { template void Net::ToHDF5(const string& filename, bool write_diff) const { - hid_t file_hid = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, - H5P_DEFAULT); - CHECK_GE(file_hid, 0) - << "Couldn't open " << filename << " to save weights."; - hid_t data_hid = H5Gcreate2(file_hid, "data", H5P_DEFAULT, H5P_DEFAULT, - H5P_DEFAULT); - CHECK_GE(data_hid, 0) << "Error saving weights to " << filename << "."; - hid_t diff_hid = -1; - if (write_diff) { - diff_hid = H5Gcreate2(file_hid, "diff", H5P_DEFAULT, H5P_DEFAULT, - H5P_DEFAULT); - CHECK_GE(diff_hid, 0) << "Error saving weights to " << filename << "."; - } - for (int layer_id = 0; layer_id < layers_.size(); ++layer_id) { - const LayerParameter& layer_param = layers_[layer_id]->layer_param(); - string layer_name = layer_param.name(); - hid_t layer_data_hid = H5Gcreate2(data_hid, layer_name.c_str(), - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - CHECK_GE(layer_data_hid, 0) - << "Error saving weights to " << filename << "."; - hid_t layer_diff_hid = -1; - if (write_diff) { - layer_diff_hid = H5Gcreate2(diff_hid, layer_name.c_str(), - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - CHECK_GE(layer_diff_hid, 0) - << "Error saving weights to " << filename << "."; - } - int num_params = layers_[layer_id]->blobs().size(); - for (int param_id = 0; param_id < num_params; ++param_id) { - ostringstream dataset_name; - dataset_name << param_id; - const int net_param_id = param_id_vecs_[layer_id][param_id]; - if (param_owners_[net_param_id] == -1) { - // Only save params that own themselves - hdf5_save_nd_dataset(layer_data_hid, dataset_name.str(), - *params_[net_param_id]); - } - if (write_diff) { - // Write diffs regardless of weight-sharing - hdf5_save_nd_dataset(layer_diff_hid, dataset_name.str(), - *params_[net_param_id], true); - } - } - H5Gclose(layer_data_hid); - if (write_diff) { - H5Gclose(layer_diff_hid); - } - } - H5Gclose(data_hid); - if (write_diff) { - H5Gclose(diff_hid); - } - H5Fclose(file_hid); + NOT_IMPLEMENTED; + //hid_t file_hid = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, + // H5P_DEFAULT); + //CHECK_GE(file_hid, 0) + // << "Couldn't open " << filename << " to save weights."; + //hid_t data_hid = H5Gcreate2(file_hid, "data", H5P_DEFAULT, H5P_DEFAULT, + // H5P_DEFAULT); + //CHECK_GE(data_hid, 0) << "Error saving weights to " << filename << "."; + //hid_t diff_hid = -1; + //if (write_diff) { + // diff_hid = H5Gcreate2(file_hid, "diff", H5P_DEFAULT, H5P_DEFAULT, + // H5P_DEFAULT); + // CHECK_GE(diff_hid, 0) << "Error saving weights to " << filename << "."; + //} + //for (int layer_id = 0; layer_id < layers_.size(); ++layer_id) { + // const LayerParameter& layer_param = layers_[layer_id]->layer_param(); + // string layer_name = layer_param.name(); + // hid_t layer_data_hid = H5Gcreate2(data_hid, layer_name.c_str(), + // H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + // CHECK_GE(layer_data_hid, 0) + // << "Error saving weights to " << filename << "."; + // hid_t layer_diff_hid = -1; + // if (write_diff) { + // layer_diff_hid = H5Gcreate2(diff_hid, layer_name.c_str(), + // H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + // CHECK_GE(layer_diff_hid, 0) + // << "Error saving weights to " << filename << "."; + // } + // int num_params = layers_[layer_id]->blobs().size(); + // for (int param_id = 0; param_id < num_params; ++param_id) { + // ostringstream dataset_name; + // dataset_name << param_id; + // const int net_param_id = param_id_vecs_[layer_id][param_id]; + // if (param_owners_[net_param_id] == -1) { + // // Only save params that own themselves + // hdf5_save_nd_dataset(layer_data_hid, dataset_name.str(), + // *params_[net_param_id]); + // } + // if (write_diff) { + // // Write diffs regardless of weight-sharing + // hdf5_save_nd_dataset(layer_diff_hid, dataset_name.str(), + // *params_[net_param_id], true); + // } + // } + // H5Gclose(layer_data_hid); + // if (write_diff) { + // H5Gclose(layer_diff_hid); + // } + //} + //H5Gclose(data_hid); + //if (write_diff) { + // H5Gclose(diff_hid); + //} + //H5Fclose(file_hid); } template diff --git a/src/caffe/solver.cpp b/src/caffe/solver.cpp index ece3913..77de6be 100644 --- a/src/caffe/solver.cpp +++ b/src/caffe/solver.cpp @@ -5,7 +5,6 @@ #include "caffe/solver.hpp" #include "caffe/util/format.hpp" -#include "caffe/util/hdf5.hpp" #include "caffe/util/io.hpp" #include "caffe/util/upgrade_proto.hpp" diff --git a/src/caffe/util/hdf5.cpp b/src/caffe/util/hdf5.cpp deleted file mode 100644 index 7730e76..0000000 --- a/src/caffe/util/hdf5.cpp +++ /dev/null @@ -1,187 +0,0 @@ -#include "caffe/util/hdf5.hpp" - -#include -#include - -namespace caffe { - -// Verifies format of data stored in HDF5 file and reshapes blob accordingly. -template -void hdf5_load_nd_dataset_helper( - hid_t file_id, const char* dataset_name_, int min_dim, int max_dim, - Blob* blob) { - // Verify that the dataset exists. - CHECK(H5LTfind_dataset(file_id, dataset_name_)) - << "Failed to find HDF5 dataset " << dataset_name_; - // Verify that the number of dimensions is in the accepted range. - herr_t status; - int ndims; - status = H5LTget_dataset_ndims(file_id, dataset_name_, &ndims); - CHECK_GE(status, 0) << "Failed to get dataset ndims for " << dataset_name_; - CHECK_GE(ndims, min_dim); - CHECK_LE(ndims, max_dim); - - // Verify that the data format is what we expect: float or double. - std::vector dims(ndims); - H5T_class_t class_; - status = H5LTget_dataset_info( - file_id, dataset_name_, dims.data(), &class_, NULL); - CHECK_GE(status, 0) << "Failed to get dataset info for " << dataset_name_; - switch (class_) { - case H5T_FLOAT: - LOG_FIRST_N(INFO, 1) << "Datatype class: H5T_FLOAT"; - break; - case H5T_INTEGER: - LOG_FIRST_N(INFO, 1) << "Datatype class: H5T_INTEGER"; - break; - case H5T_TIME: - LOG(FATAL) << "Unsupported datatype class: H5T_TIME"; - case H5T_STRING: - LOG(FATAL) << "Unsupported datatype class: H5T_STRING"; - case H5T_BITFIELD: - LOG(FATAL) << "Unsupported datatype class: H5T_BITFIELD"; - case H5T_OPAQUE: - LOG(FATAL) << "Unsupported datatype class: H5T_OPAQUE"; - case H5T_COMPOUND: - LOG(FATAL) << "Unsupported datatype class: H5T_COMPOUND"; - case H5T_REFERENCE: - LOG(FATAL) << "Unsupported datatype class: H5T_REFERENCE"; - case H5T_ENUM: - LOG(FATAL) << "Unsupported datatype class: H5T_ENUM"; - case H5T_VLEN: - LOG(FATAL) << "Unsupported datatype class: H5T_VLEN"; - case H5T_ARRAY: - LOG(FATAL) << "Unsupported datatype class: H5T_ARRAY"; - default: - LOG(FATAL) << "Datatype class unknown"; - } - - vector blob_dims(dims.size()); - for (int i = 0; i < dims.size(); ++i) { - blob_dims[i] = dims[i]; - } - blob->Reshape(blob_dims); -} - -template <> -void hdf5_load_nd_dataset(hid_t file_id, const char* dataset_name_, - int min_dim, int max_dim, Blob* blob) { - hdf5_load_nd_dataset_helper(file_id, dataset_name_, min_dim, max_dim, blob); - herr_t status = H5LTread_dataset_float( - file_id, dataset_name_, blob->mutable_cpu_data()); - CHECK_GE(status, 0) << "Failed to read float dataset " << dataset_name_; -} - -template <> -void hdf5_load_nd_dataset(hid_t file_id, const char* dataset_name_, - int min_dim, int max_dim, Blob* blob) { - hdf5_load_nd_dataset_helper(file_id, dataset_name_, min_dim, max_dim, blob); - herr_t status = H5LTread_dataset_double( - file_id, dataset_name_, blob->mutable_cpu_data()); - CHECK_GE(status, 0) << "Failed to read double dataset " << dataset_name_; -} - -template <> -void hdf5_save_nd_dataset( - const hid_t file_id, const string& dataset_name, const Blob& blob, - bool write_diff) { - int num_axes = blob.num_axes(); - hsize_t *dims = new hsize_t[num_axes]; - for (int i = 0; i < num_axes; ++i) { - dims[i] = blob.shape(i); - } - const float* data; - if (write_diff) { - data = blob.cpu_diff(); - } else { - data = blob.cpu_data(); - } - herr_t status = H5LTmake_dataset_float( - file_id, dataset_name.c_str(), num_axes, dims, data); - CHECK_GE(status, 0) << "Failed to make float dataset " << dataset_name; - delete[] dims; -} - -template <> -void hdf5_save_nd_dataset( - hid_t file_id, const string& dataset_name, const Blob& blob, - bool write_diff) { - int num_axes = blob.num_axes(); - hsize_t *dims = new hsize_t[num_axes]; - for (int i = 0; i < num_axes; ++i) { - dims[i] = blob.shape(i); - } - const double* data; - if (write_diff) { - data = blob.cpu_diff(); - } else { - data = blob.cpu_data(); - } - herr_t status = H5LTmake_dataset_double( - file_id, dataset_name.c_str(), num_axes, dims, data); - CHECK_GE(status, 0) << "Failed to make double dataset " << dataset_name; - delete[] dims; -} - -string hdf5_load_string(hid_t loc_id, const string& dataset_name) { - // Get size of dataset - size_t size; - H5T_class_t class_; - herr_t status = \ - H5LTget_dataset_info(loc_id, dataset_name.c_str(), NULL, &class_, &size); - CHECK_GE(status, 0) << "Failed to get dataset info for " << dataset_name; - char *buf = new char[size]; - status = H5LTread_dataset_string(loc_id, dataset_name.c_str(), buf); - CHECK_GE(status, 0) - << "Failed to load int dataset with name " << dataset_name; - string val(buf); - delete[] buf; - return val; -} - -void hdf5_save_string(hid_t loc_id, const string& dataset_name, - const string& s) { - herr_t status = \ - H5LTmake_dataset_string(loc_id, dataset_name.c_str(), s.c_str()); - CHECK_GE(status, 0) - << "Failed to save string dataset with name " << dataset_name; -} - -int hdf5_load_int(hid_t loc_id, const string& dataset_name) { - int val; - herr_t status = H5LTread_dataset_int(loc_id, dataset_name.c_str(), &val); - CHECK_GE(status, 0) - << "Failed to load int dataset with name " << dataset_name; - return val; -} - -void hdf5_save_int(hid_t loc_id, const string& dataset_name, int i) { - hsize_t one = 1; - herr_t status = \ - H5LTmake_dataset_int(loc_id, dataset_name.c_str(), 1, &one, &i); - CHECK_GE(status, 0) - << "Failed to save int dataset with name " << dataset_name; -} - -int hdf5_get_num_links(hid_t loc_id) { - H5G_info_t info; - herr_t status = H5Gget_info(loc_id, &info); - CHECK_GE(status, 0) << "Error while counting HDF5 links."; - return info.nlinks; -} - -string hdf5_get_name_by_idx(hid_t loc_id, int idx) { - ssize_t str_size = H5Lget_name_by_idx( - loc_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, idx, NULL, 0, H5P_DEFAULT); - CHECK_GE(str_size, 0) << "Error retrieving HDF5 dataset at index " << idx; - char *c_str = new char[str_size+1]; - ssize_t status = H5Lget_name_by_idx( - loc_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, idx, c_str, str_size+1, - H5P_DEFAULT); - CHECK_GE(status, 0) << "Error retrieving HDF5 dataset at index " << idx; - string result(c_str); - delete[] c_str; - return result; -} - -} // namespace caffe diff --git a/src/caffe/util/io.cpp b/src/caffe/util/io.cpp index 835d2d4..b2f9b48 100644 --- a/src/caffe/util/io.cpp +++ b/src/caffe/util/io.cpp @@ -19,6 +19,9 @@ #include "caffe/proto/caffe.pb.h" #include "caffe/util/io.hpp" +#include +#define open _open + const int kProtoReadBytesLimit = INT_MAX; // Max size of 2 GB minus 1 byte. namespace caffe { @@ -50,7 +53,7 @@ void WriteProtoToTextFile(const Message& proto, const char* filename) { } bool ReadProtoFromBinaryFile(const char* filename, Message* proto) { - int fd = open(filename, O_RDONLY); + int fd = open(filename, O_RDONLY | O_BINARY); CHECK_NE(fd, -1) << "File not found: " << filename; ZeroCopyInputStream* raw_input = new FileInputStream(fd); CodedInputStream* coded_input = new CodedInputStream(raw_input); diff --git a/src/caffe/util/signal_handler.cpp b/src/caffe/util/signal_handler.cpp index 5d764ec..c3539f6 100644 --- a/src/caffe/util/signal_handler.cpp +++ b/src/caffe/util/signal_handler.cpp @@ -6,110 +6,110 @@ #include "caffe/util/signal_handler.h" -namespace { - static volatile sig_atomic_t got_sigint = false; - static volatile sig_atomic_t got_sighup = false; - static bool already_hooked_up = false; - - void handle_signal(int signal) { - switch (signal) { - case SIGHUP: - got_sighup = true; - break; - case SIGINT: - got_sigint = true; - break; - } - } - - void HookupHandler() { - if (already_hooked_up) { - LOG(FATAL) << "Tried to hookup signal handlers more than once."; - } - already_hooked_up = true; - - struct sigaction sa; - // Setup the handler - sa.sa_handler = &handle_signal; - // Restart the system call, if at all possible - sa.sa_flags = SA_RESTART; - // Block every signal during the handler - sigfillset(&sa.sa_mask); - // Intercept SIGHUP and SIGINT - if (sigaction(SIGHUP, &sa, NULL) == -1) { - LOG(FATAL) << "Cannot install SIGHUP handler."; - } - if (sigaction(SIGINT, &sa, NULL) == -1) { - LOG(FATAL) << "Cannot install SIGINT handler."; - } - } - - // Set the signal handlers to the default. - void UnhookHandler() { - if (already_hooked_up) { - struct sigaction sa; - // Setup the sighub handler - sa.sa_handler = SIG_DFL; - // Restart the system call, if at all possible - sa.sa_flags = SA_RESTART; - // Block every signal during the handler - sigfillset(&sa.sa_mask); - // Intercept SIGHUP and SIGINT - if (sigaction(SIGHUP, &sa, NULL) == -1) { - LOG(FATAL) << "Cannot uninstall SIGHUP handler."; - } - if (sigaction(SIGINT, &sa, NULL) == -1) { - LOG(FATAL) << "Cannot uninstall SIGINT handler."; - } - - already_hooked_up = false; - } - } - - // Return true iff a SIGINT has been received since the last time this - // function was called. - bool GotSIGINT() { - bool result = got_sigint; - got_sigint = false; - return result; - } - - // Return true iff a SIGHUP has been received since the last time this - // function was called. - bool GotSIGHUP() { - bool result = got_sighup; - got_sighup = false; - return result; - } -} // namespace - -namespace caffe { - -SignalHandler::SignalHandler(SolverAction::Enum SIGINT_action, - SolverAction::Enum SIGHUP_action): - SIGINT_action_(SIGINT_action), - SIGHUP_action_(SIGHUP_action) { - HookupHandler(); -} - -SignalHandler::~SignalHandler() { - UnhookHandler(); -} - -SolverAction::Enum SignalHandler::CheckForSignals() const { - if (GotSIGHUP()) { - return SIGHUP_action_; - } - if (GotSIGINT()) { - return SIGINT_action_; - } - return SolverAction::NONE; -} - -// Return the function that the solver can use to find out if a snapshot or -// early exit is being requested. -ActionCallback SignalHandler::GetActionFunction() { - return boost::bind(&SignalHandler::CheckForSignals, this); -} - -} // namespace caffe +//namespace { +// static volatile sig_atomic_t got_sigint = false; +// static volatile sig_atomic_t got_sighup = false; +// static bool already_hooked_up = false; +// +// void handle_signal(int signal) { +// switch (signal) { +// case SIGHUP: +// got_sighup = true; +// break; +// case SIGINT: +// got_sigint = true; +// break; +// } +// } +// +// void HookupHandler() { +// if (already_hooked_up) { +// LOG(FATAL) << "Tried to hookup signal handlers more than once."; +// } +// already_hooked_up = true; +// +// struct sigaction sa; +// // Setup the handler +// sa.sa_handler = &handle_signal; +// // Restart the system call, if at all possible +// sa.sa_flags = SA_RESTART; +// // Block every signal during the handler +// sigfillset(&sa.sa_mask); +// // Intercept SIGHUP and SIGINT +// if (sigaction(SIGHUP, &sa, NULL) == -1) { +// LOG(FATAL) << "Cannot install SIGHUP handler."; +// } +// if (sigaction(SIGINT, &sa, NULL) == -1) { +// LOG(FATAL) << "Cannot install SIGINT handler."; +// } +// } +// +// // Set the signal handlers to the default. +// void UnhookHandler() { +// if (already_hooked_up) { +// struct sigaction sa; +// // Setup the sighub handler +// sa.sa_handler = SIG_DFL; +// // Restart the system call, if at all possible +// sa.sa_flags = SA_RESTART; +// // Block every signal during the handler +// sigfillset(&sa.sa_mask); +// // Intercept SIGHUP and SIGINT +// if (sigaction(SIGHUP, &sa, NULL) == -1) { +// LOG(FATAL) << "Cannot uninstall SIGHUP handler."; +// } +// if (sigaction(SIGINT, &sa, NULL) == -1) { +// LOG(FATAL) << "Cannot uninstall SIGINT handler."; +// } +// +// already_hooked_up = false; +// } +// } +// +// // Return true iff a SIGINT has been received since the last time this +// // function was called. +// bool GotSIGINT() { +// bool result = got_sigint; +// got_sigint = false; +// return result; +// } +// +// // Return true iff a SIGHUP has been received since the last time this +// // function was called. +// bool GotSIGHUP() { +// bool result = got_sighup; +// got_sighup = false; +// return result; +// } +//} // namespace +// +//namespace caffe { +// +//SignalHandler::SignalHandler(SolverAction::Enum SIGINT_action, +// SolverAction::Enum SIGHUP_action): +// SIGINT_action_(SIGINT_action), +// SIGHUP_action_(SIGHUP_action) { +// HookupHandler(); +//} +// +//SignalHandler::~SignalHandler() { +// UnhookHandler(); +//} +// +//SolverAction::Enum SignalHandler::CheckForSignals() const { +// if (GotSIGHUP()) { +// return SIGHUP_action_; +// } +// if (GotSIGINT()) { +// return SIGINT_action_; +// } +// return SolverAction::NONE; +//} +// +//// Return the function that the solver can use to find out if a snapshot or +//// early exit is being requested. +//ActionCallback SignalHandler::GetActionFunction() { +// return boost::bind(&SignalHandler::CheckForSignals, this); +//} +// +//} // namespace caffe