Skip to content

Commit

Permalink
minor
Browse files Browse the repository at this point in the history
  • Loading branch information
@roccrtx
roccrtx committed Nov 12, 2018
1 parent 60356ec commit 9f4416f
Show file tree
Hide file tree
Showing 7 changed files with 86 additions and 72 deletions.
38 changes: 19 additions & 19 deletions logging.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,38 +10,38 @@
namespace rdmaio {

/**
* \def LOG_FATAL
* \def FATAL
* Used for fatal and probably irrecoverable conditions
* \def LOG_ERROR
* \def ERROR
* Used for errors which are recoverable within the scope of the function
* \def LOG_WARNING
* \def WARNING
* Logs interesting conditions which are probably not fatal
* \def LOG_EMPH
* Outputs as LOG_INFO, but in LOG_WARNING colors. Useful for
* \def EMPH
* Outputs as INFO, but in WARNING colors. Useful for
* outputting information you want to emphasize.
* \def LOG_INFO
* \def INFO
* Used for providing general useful information
* \def LOG_DEBUG
* \def DEBUG
* Debugging purposes only
* \def LOG_EVERYTHING
* \def EVERYTHING
* Log everything
*/

enum loglevel {
LOG_NONE = 7,
LOG_FATAL = 6,
LOG_ERROR = 5,
LOG_WARNING = 4,
LOG_EMPH = 3,
LOG_INFO = 2,
LOG_DEBUG = 1,
LOG_EVERYTHING = 0
NONE = 7,
FATAL = 6,
ERROR = 5,
WARNING = 4,
EMPH = 3,
INFO = 2,
DEBUG = 1,
EVERYTHING = 0
};

#define unlikely(x) __builtin_expect(!!(x), 0)

#ifndef RDMA_LOG_LEVEL
#define RDMA_LOG_LEVEL ::rdmaio::LOG_INFO
#define RDMA_LOG_LEVEL ::rdmaio::INFO
#endif

// logging macro definiations
Expand All @@ -62,7 +62,7 @@ enum loglevel {

#define RDMA_ASSERT(condition) \
if(unlikely(!(condition))) \
::rdmaio::MessageLogger((char*)__FILE__, __LINE__, ::rdmaio::LOG_FATAL + 1).stream() << "Assertion! "
::rdmaio::MessageLogger((char*)__FILE__, __LINE__, ::rdmaio::FATAL + 1).stream() << "Assertion! "

#define RDMA_VERIFY(n,condition) RDMA_LOG_IF(n,(!(condition)))

Expand All @@ -79,7 +79,7 @@ class MessageLogger {
stream_ << "\n";
std::cout << "\033[" << RDMA_DEBUG_LEVEL_COLOR[std::min(level_,6)] << "m"
<< stream_.str() << EndcolorFlag();
if(level_ >= ::rdmaio::LOG_FATAL)
if(level_ >= ::rdmaio::FATAL)
abort();
}
}
Expand Down
4 changes: 2 additions & 2 deletions mr.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ class Memory {
mr(ibv_reg_mr(pd,addr,len,flag))
{
if(mr == nullptr) {
RDMA_LOG(LOG_WARNING) << "failed to register mr, for addr " << addr << "; len " << len;
RDMA_LOG(WARNING) << "failed to register mr, for addr " << addr << "; len " << len;
} else {
rattr.buf = (uintptr_t)addr;
rattr.key = mr->rkey;
Expand All @@ -35,7 +35,7 @@ class Memory {
~Memory() {
if(mr != nullptr) {
int rc = ibv_dereg_mr(mr);
RDMA_LOG_IF(LOG_ERROR,rc != 0) << "dereg mr error: " << strerror(errno);
RDMA_LOG_IF(ERROR,rc != 0) << "dereg mr error: " << strerror(errno);
}
}

Expand Down
9 changes: 5 additions & 4 deletions qp.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,8 @@ class RRCQP : public QP {
// first check whether QP is valid to connect
enum ibv_qp_state state;
if( (state = QPImpl::query_qp_status(qp_)) != IBV_QPS_INIT) {
RDMA_LOG(LOG_WARNING) << "qp not in a connect state to connect!";
if(state != IBV_QPS_RTS)
RDMA_LOG(WARNING) << "qp not in a correct state to connect!";
return (state == IBV_QPS_RTS)?SUCC:UNKNOWN;
}
ConnArg arg = {} ; ConnReply reply = {};
Expand All @@ -172,13 +173,13 @@ class RRCQP : public QP {
if(ret == SUCC) {
// change QP status
if(!RCQPImpl::ready2rcv<F>(qp_,reply.payload.qp,rnic_)) {
RDMA_LOG(LOG_WARNING) << "change qp status to ready to receive error: " << strerror(errno);
RDMA_LOG(WARNING) << "change qp status to ready to receive error: " << strerror(errno);
ret = ERR;
goto CONN_END;
}

if(!RCQPImpl::ready2send<F>(qp_)) {
RDMA_LOG(LOG_WARNING) << "change qp status to ready to send error: " << strerror(errno);
RDMA_LOG(WARNING) << "change qp status to ready to send error: " << strerror(errno);
ret = ERR;
goto CONN_END;
}
Expand Down Expand Up @@ -387,7 +388,7 @@ class RUDQP : public QP {
// create the ah, and store the address handler
auto ah = UDQPImpl::create_ah(rnic_,reply.payload.qp);
if(ah == nullptr) {
RDMA_LOG(LOG_WARNING) << "create address handler error: " << strerror(errno);
RDMA_LOG(WARNING) << "create address handler error: " << strerror(errno);
ret = ERR;
} else {
ahs_[reply.payload.qp.node_id] = ah;
Expand Down
21 changes: 11 additions & 10 deletions qp_impl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ class QPImpl {

auto socket = PreConnector::get_send_socket(ip,port);
if(socket < 0) {
RDMA_LOG(3) << "get socket err";
return ERR;
}

Expand Down Expand Up @@ -155,11 +156,11 @@ class RCQPImpl {

int flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS;
int rc = ibv_modify_qp(qp, &qp_attr,flags);
RDMA_VERIFY(LOG_WARNING,rc == 0) << "Failed to modify RC to INIT state, %s\n" << strerror(errno);
RDMA_VERIFY(WARNING,rc == 0) << "Failed to modify RC to INIT state, %s\n" << strerror(errno);

if(rc != 0) {
// error handling
RDMA_LOG(LOG_WARNING) << " change state to init failed. ";
RDMA_LOG(WARNING) << " change state to init failed. ";
}
}

Expand Down Expand Up @@ -223,7 +224,7 @@ class RCQPImpl {

// create the CQ
cq = ibv_create_cq(rnic->ctx, RC_MAX_SEND_SIZE, nullptr, nullptr, 0);
RDMA_VERIFY(LOG_WARNING,cq != nullptr) << "create cq error: " << strerror(errno);
RDMA_VERIFY(WARNING,cq != nullptr) << "create cq error: " << strerror(errno);

// create the QP
struct ibv_qp_init_attr qp_init_attr = {};
Expand All @@ -239,7 +240,7 @@ class RCQPImpl {
qp_init_attr.cap.max_inline_data = MAX_INLINE_SIZE;

qp = ibv_create_qp(rnic->pd, &qp_init_attr);
RDMA_VERIFY(LOG_WARNING,qp != nullptr);
RDMA_VERIFY(WARNING,qp != nullptr);

if(qp)
ready2init<F>(qp,rnic);
Expand All @@ -265,12 +266,12 @@ class UDQPImpl {
return;

if((cq = ibv_create_cq(rnic->ctx, config.max_send_size, nullptr, nullptr, 0)) == nullptr) {
RDMA_LOG(LOG_ERROR) << "create send cq for UD QP error: " << strerror(errno);
RDMA_LOG(ERROR) << "create send cq for UD QP error: " << strerror(errno);
return;
}

if((recv_cq = ibv_create_cq(rnic->ctx, config.max_recv_size, nullptr, nullptr, 0)) == nullptr) {
RDMA_LOG(LOG_ERROR) << "create recv cq for UD QP error: " << strerror(errno);
RDMA_LOG(ERROR) << "create recv cq for UD QP error: " << strerror(errno);
return;
}

Expand All @@ -287,18 +288,18 @@ class UDQPImpl {
qp_init_attr.cap.max_inline_data = MAX_INLINE_SIZE;

if((qp = ibv_create_qp(rnic->pd, &qp_init_attr)) == nullptr) {
RDMA_LOG(LOG_ERROR) << "create send qp for UD QP error: " << strerror(errno);
RDMA_LOG(ERROR) << "create send qp for UD QP error: " << strerror(errno);
return;
}

// change QP status
ready2init(qp, rnic,config); // shall always succeed

if(!ready2rcv(qp,rnic)) {
RDMA_LOG(LOG_WARNING) << "change ud qp to ready to recv error: " << strerror(errno);
RDMA_LOG(WARNING) << "change ud qp to ready to recv error: " << strerror(errno);
}
if(!ready2send(qp,config)) {
RDMA_LOG(LOG_WARNING) << "change ud qp to ready to send error: " << strerror(errno);
RDMA_LOG(WARNING) << "change ud qp to ready to send error: " << strerror(errno);
}
}

Expand All @@ -317,7 +318,7 @@ class UDQPImpl {
qp_attr.qkey = config.qkey;

if((rc = ibv_modify_qp(qp, &qp_attr, flags)) != 0) {
RDMA_LOG(LOG_WARNING) << "modify ud qp to init error: " << strerror(errno);
RDMA_LOG(WARNING) << "modify ud qp to init error: " << strerror(errno);
}
}

Expand Down
2 changes: 2 additions & 0 deletions rdma_ctrl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ class RdmaCtrl {
*/
std::vector<RNicInfo> query_devs();

static std::vector<RNicInfo> query_devs_helper();

// clear the cached infos by RdmaCtrl;
void clear_dev_info();

Expand Down
64 changes: 37 additions & 27 deletions rdma_ctrl_impl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ class RdmaCtrl::RdmaCtrlImpl {
~RdmaCtrlImpl() {
running_ = false; // wait for the handler to join
pthread_join(handler_tid_,NULL);
RDMA_LOG(LOG_INFO) << "rdma controler close: does not handle any future connections.";
RDMA_LOG(INFO) << "rdma controler close: does not handle any future connections.";
}

RNicHandler *open_thread_local_device(DevIdx idx) {
Expand All @@ -79,32 +79,32 @@ class RdmaCtrl::RdmaCtrlImpl {
dev_list = ibv_get_device_list(&num_devices);

if(idx.dev_id >= num_devices || idx.dev_id < 0) {
RDMA_LOG(LOG_WARNING) << "wrong dev_id: " << idx.dev_id << "; total " << num_devices <<" found";
RDMA_LOG(WARNING) << "wrong dev_id: " << idx.dev_id << "; total " << num_devices <<" found";
goto OPEN_END;
}

// alloc ctx
ib_ctx = ibv_open_device(dev_list[idx.dev_id]);
if(ib_ctx == nullptr) {
RDMA_LOG(LOG_WARNING) << "failed to open ib ctx w error: " << strerror(errno);
RDMA_LOG(WARNING) << "failed to open ib ctx w error: " << strerror(errno);
goto OPEN_END;
}

// alloc pd
pd = ibv_alloc_pd(ib_ctx);
if(pd == nullptr) {
RDMA_LOG(LOG_WARNING) << "failed to alloc pd w error: " << strerror(errno);
RDMA_VERIFY(LOG_INFO,ibv_close_device(ib_ctx) == 0) << "failed to close device " << idx.dev_id;
RDMA_LOG(WARNING) << "failed to alloc pd w error: " << strerror(errno);
RDMA_VERIFY(INFO,ibv_close_device(ib_ctx) == 0) << "failed to close device " << idx.dev_id;
goto OPEN_END;
}

// fill the lid
ibv_port_attr port_attr;
rc = ibv_query_port (ib_ctx, idx.port_id, &port_attr);
if(rc < 0) {
RDMA_LOG(LOG_WARNING) << "failed to query port status w error: " << strerror(errno);
RDMA_VERIFY(LOG_INFO,ibv_close_device(ib_ctx) == 0) << "failed to close device " << idx.dev_id;
RDMA_VERIFY(LOG_INFO,ibv_dealloc_pd(pd) == 0) << "failed to dealloc pd";
RDMA_LOG(WARNING) << "failed to query port status w error: " << strerror(errno);
RDMA_VERIFY(INFO,ibv_close_device(ib_ctx) == 0) << "failed to close device " << idx.dev_id;
RDMA_VERIFY(INFO,ibv_dealloc_pd(pd) == 0) << "failed to dealloc pd";
goto OPEN_END;
}

Expand Down Expand Up @@ -193,14 +193,14 @@ class RdmaCtrl::RdmaCtrlImpl {

Memory *m = new Memory(buf,size,rnic->pd,flag);
if(!m->valid()) {
RDMA_LOG(LOG_WARNING) << "register mr to rnic error: " << strerror(errno);
RDMA_LOG(WARNING) << "register mr to rnic error: " << strerror(errno);
delete m;
return false;
}
{
SCS s;
if(mrs_.find(mr_id) != mrs_.end()) {
RDMA_LOG(LOG_WARNING) << "mr " << mr_id << " has already been registered!";
RDMA_LOG(WARNING) << "mr " << mr_id << " has already been registered!";
delete m;
} else {
mrs_.insert(std::make_pair(mr_id,m));
Expand Down Expand Up @@ -232,21 +232,16 @@ class RdmaCtrl::RdmaCtrlImpl {
cached_infos_.clear();
}

std::vector<RNicInfo> query_devs() {

static std::vector<RNicInfo> query_devs_helper() {
int num_devices = 0; struct ibv_device **dev_list = nullptr;
std::vector<RNicInfo> res;

if(cached_infos_.size() != 0) {
//RDMA_LOG(LOG_INFO) << "use cached device info. If not wanted, use clear_dev_info(); ";
num_devices = cached_infos_.size();
goto QUERY_END;
}
{ // query the device and its active ports using the underlying APIs
dev_list = ibv_get_device_list(&num_devices);
int temp_devices = num_devices;

if(dev_list == nullptr) {
RDMA_LOG(LOG_ERROR) << "cannot get ib devices.";
RDMA_LOG(ERROR) << "cannot get ib devices.";
num_devices = 0;
goto QUERY_END;
}
Expand All @@ -255,21 +250,31 @@ class RdmaCtrl::RdmaCtrlImpl {

struct ibv_context *ib_ctx = ibv_open_device(dev_list[dev_id]);
if(ib_ctx == nullptr) {
RDMA_LOG(LOG_ERROR) << "open dev " << dev_id << " error: " << strerror(errno) << " ignored";
RDMA_LOG(ERROR) << "open dev " << dev_id << " error: " << strerror(errno) << " ignored";
num_devices -= 1;
continue;
}
cached_infos_.emplace_back(ibv_get_device_name(ib_ctx->device),dev_id,ib_ctx);
res.emplace_back(ibv_get_device_name(ib_ctx->device),dev_id,ib_ctx);
QUERY_DEV_END:
// close ib_ctx
RDMA_VERIFY(LOG_INFO,ibv_close_device(ib_ctx) == 0) << "failed to close device " << dev_id;
RDMA_VERIFY(INFO,ibv_close_device(ib_ctx) == 0) << "failed to close device " << dev_id;
}
}

QUERY_END:
if(dev_list != nullptr)
ibv_free_device_list(dev_list);
return std::vector<RNicInfo>(cached_infos_.begin(),cached_infos_.end());
return res;
}

std::vector<RNicInfo> query_devs() {


if(cached_infos_.size() != 0) {
//RDMA_LOG(INFO) << "use cached device info. If not wanted, use clear_dev_info(); ";
return cached_infos_;
}
return query_devs_helper();
}

RdmaCtrl::DevIdx convert_port_idx(int idx) {
Expand Down Expand Up @@ -322,9 +327,9 @@ class RdmaCtrl::RdmaCtrlImpl {
auto listenfd = PreConnector::get_listen_socket(local_ip_,tcp_base_port_);

int opt = 1;
RDMA_VERIFY(LOG_ERROR,setsockopt(listenfd,SOL_SOCKET,SO_REUSEADDR | SO_REUSEPORT,&opt,sizeof(int)) == 0)
RDMA_VERIFY(ERROR,setsockopt(listenfd,SOL_SOCKET,SO_REUSEADDR | SO_REUSEPORT,&opt,sizeof(int)) == 0)
<< "unable to configure socket status.";
RDMA_VERIFY(LOG_ERROR,listen(listenfd,24) == 0) << "TCP listen error: " << strerror(errno);
RDMA_VERIFY(ERROR,listen(listenfd,24) == 0) << "TCP listen error: " << strerror(errno);

while(running_) {

Expand All @@ -335,7 +340,7 @@ class RdmaCtrl::RdmaCtrlImpl {
auto csfd = accept(listenfd,(struct sockaddr *) &cli_addr, &clilen);

if(csfd < 0) {
RDMA_LOG(LOG_ERROR) << "accept a wrong connection error: " << strerror(errno);
RDMA_LOG(ERROR) << "accept a wrong connection error: " << strerror(errno);
continue;
}

Expand Down Expand Up @@ -385,7 +390,7 @@ class RdmaCtrl::RdmaCtrlImpl {
}
break;
default:
RDMA_LOG(LOG_ERROR) << "unknown QP connection type: " << arg.payload.qp.qp_type;
RDMA_LOG(ERROR) << "unknown QP connection type: " << arg.payload.qp.qp_type;
}
if(qp != nullptr) {
reply.payload.qp = qp->get_attr();
Expand All @@ -395,7 +400,7 @@ class RdmaCtrl::RdmaCtrlImpl {
break;
}
default:
RDMA_LOG(LOG_WARNING) << "received unknown connect type " << arg.type;
RDMA_LOG(WARNING) << "received unknown connect type " << arg.type;
}
} // end simple critical section protection

Expand Down Expand Up @@ -581,4 +586,9 @@ bool RdmaCtrl::link_symmetric_rcqps(const std::vector<std::string> &cluster,
return impl_->link_symmetric_rcqps(cluster,l_mrid,mr_id,wid,idx);
}

inline __attribute__ ((always_inline))
std::vector<RNicInfo> RdmaCtrl::query_devs_helper() {
return RdmaCtrlImpl::query_devs_helper();
}

};
Loading

0 comments on commit 9f4416f

Please sign in to comment.