sum_blob.set_channels(datum.channels());
sum_blob.set_height(datum.height());
sum_blob.set_width(datum.width());
+ const int data_size = datum.channels() * datum.height() * datum.width();
for (int i = 0; i < datum.data().size(); ++i) {
sum_blob.add_data(0.);
}
// just a dummy operation
datum.ParseFromString(it->value().ToString());
const string& data = datum.data();
+ CHECK_EQ(data.size(), data_size) << "Incorrect data field size " << data.size();
for (int i = 0; i < data.size(); ++i) {
sum_blob.set_data(i, sum_blob.data(i) + (uint8_t)data[i]);
}
LOG(INFO) << "Restoring previous solver status from " << resume_file;
Solver<Dtype>::Restore(resume_file);
}
+ next_snapshot_ = this->iter_ + this->param_.snapshot();
// the main loop.
LOG(INFO) << "Waiting for incoming updates...";
total_received += count;
}
LOG(INFO) << "Received " << total_received << " variables.";
- // Check Error
- if (!data_stream) {
- LOG(ERROR) << "Error in receiving.";
+ // Check error: if there are any error in the receiving phase, we will not
+ // trust the passed in update.
+ if (data_stream.error()) {
+ LOG(ERROR) << "Error in receiving. Error code: " << data_stream.error().message();
} else {
// If the read is successful, update the network.
this->iter_ += incoming_iter;
}
LOG(INFO) << "Sent " << total_sent << " variables.";
data_stream.flush();
- data_stream.close();
}
// For a network that is trained by the solver, no bottom or top vecs
// should be given, and we will just provide dummy vecs.
vector<Blob<Dtype>*> bottom_vec;
+ next_display_ = this->iter_ + this->param_.display();
while (this->iter_++ < this->param_.max_iter()) {
Dtype loss = this->net_->ForwardBackward(bottom_vec);
ComputeUpdateValue();
if (this->param_.display() && this->iter_ > next_display_) {
LOG(INFO) << "Iteration " << this->iter_ << ", loss = " << loss;
- next_display_ += this->param_.display();
+ next_display_ = this->iter_ + this->param_.display();
}
}
LOG(INFO) << "Optimization Done.";
template <typename Dtype>
void DistributedSolverParamClient<Dtype>::SendAndReceive(bool receive_only) {
tcp::iostream data_stream(this->param_.tcp_server(), this->param_.tcp_port());
- CHECK(data_stream) << "Error in connection.";
+ if (!data_stream) {
+ LOG(FATAL) << "Unable to connect. Error code: " << data_stream.error().message();
+ }
data_stream.write(reinterpret_cast<char*>(&receive_only), sizeof(receive_only));
if (!receive_only) {
LOG(INFO) << "Sending local changes.";
total_sent += count;
}
LOG(INFO) << "Sent " << total_sent << " variables.";
+ CHECK(!data_stream.error()) << "Error in sending. Error code: "
+ << data_stream.error().message();
}// else {
// LOG(INFO) << "Not sending local changes. Receive only.";
//}
memset(net_params[param_id]->mutable_cpu_diff(), 0,
net_params[param_id]->count() * sizeof(Dtype));
}
+ CHECK(!data_stream.error()) << "Error in communication. Error code: "
+ << data_stream.error().message();
LOG(INFO) << "Received " << total_received << " variables.";
// Set the next send iter.
next_send_iter_ = this->iter_ + this->param_.communication_interval();
}
-void ReadImageToDatum(const string& filename, const int label, Datum* datum) {
+bool ReadImageToDatum(const string& filename, const int label, Datum* datum) {
cv::Mat cv_img;
cv_img = cv::imread(filename, CV_LOAD_IMAGE_COLOR);
- CHECK(cv_img.data) << "Could not open or find the image.";
+ if (!cv_img.data) {
+ LOG(ERROR) << "Could not open or find file " << filename;
+ return false;
+ }
datum->set_channels(3);
datum->set_height(cv_img.rows);
datum->set_width(cv_img.cols);
}
}
}
+ return true;
}
} // namespace caffe