-// pycaffe provides a wrapper of the caffe::Net class as well as some
-// caffe::Caffe functions so that one could easily call it from Python.
-// Note that for Python, we will simply use float as the data type.
-
#include <Python.h> // NOLINT(build/include_alpha)
// Produce deprecation warnings (needs to come before arrayobject.h inclusion).
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#include <boost/make_shared.hpp>
+#include <boost/python.hpp>
#include <boost/python/suite/indexing/vector_indexing_suite.hpp>
+#include <numpy/arrayobject.h>
// these need to be included after boost on OS X
#include <string> // NOLINT(build/include_order)
#include <vector> // NOLINT(build/include_order)
#include <fstream> // NOLINT
-#include "_caffe.hpp"
#include "caffe/caffe.hpp"
// Temporary solution for numpy < 1.7 versions: old macro, no promises.
namespace caffe {
-// for convenience, check that input files can be opened, and raise an
+// For Python, for now, we'll just always use float as the type.
+typedef float Dtype;
+
+// For convenience, check that input files can be opened, and raise an
// exception that boost will send to Python if not (caffe could still crash
// later if the input files are disturbed before they are actually used, but
-// this saves frustration in most cases)
+// this saves frustration in most cases).
static void CheckFile(const string& filename) {
std::ifstream f(filename.c_str());
if (!f.good()) {
f.close();
}
-bp::object PyBlobWrap::get_data() {
- npy_intp dims[] = {num(), channels(), height(), width()};
-
- PyObject *obj = PyArray_SimpleNewFromData(4, dims, NPY_FLOAT32,
- blob_->mutable_cpu_data());
- PyArray_SetBaseObject(reinterpret_cast<PyArrayObject *>(obj), self_);
- Py_INCREF(self_);
- bp::handle<> h(obj);
-
- return bp::object(h);
-}
-
-bp::object PyBlobWrap::get_diff() {
- npy_intp dims[] = {num(), channels(), height(), width()};
-
- PyObject *obj = PyArray_SimpleNewFromData(4, dims, NPY_FLOAT32,
- blob_->mutable_cpu_diff());
- PyArray_SetBaseObject(reinterpret_cast<PyArrayObject *>(obj), self_);
- Py_INCREF(self_);
- bp::handle<> h(obj);
-
- return bp::object(h);
-}
-
-PyNet::PyNet(string param_file, string pretrained_param_file) {
- Init(param_file);
- CheckFile(pretrained_param_file);
- net_->CopyTrainedLayersFrom(pretrained_param_file);
-}
-
-void PyNet::Init(string param_file) {
- CheckFile(param_file);
- net_.reset(new Net<float>(param_file));
-}
-
-void PyNet::check_contiguous_array(PyArrayObject* arr, string name,
- int channels, int height, int width) {
- if (!(PyArray_FLAGS(arr) & NPY_ARRAY_C_CONTIGUOUS)) {
- throw std::runtime_error(name + " must be C contiguous");
- }
- if (PyArray_NDIM(arr) != 4) {
- throw std::runtime_error(name + " must be 4-d");
- }
- if (PyArray_TYPE(arr) != NPY_FLOAT32) {
- throw std::runtime_error(name + " must be float32");
- }
- if (PyArray_DIMS(arr)[1] != channels) {
- throw std::runtime_error(name + " has wrong number of channels");
- }
- if (PyArray_DIMS(arr)[2] != height) {
- throw std::runtime_error(name + " has wrong height");
- }
- if (PyArray_DIMS(arr)[3] != width) {
- throw std::runtime_error(name + " has wrong width");
- }
-}
-
-void PyNet::set_input_arrays(bp::object data_obj, bp::object labels_obj) {
- // check that this network has an input MemoryDataLayer
- shared_ptr<MemoryDataLayer<float> > md_layer =
- boost::dynamic_pointer_cast<MemoryDataLayer<float> >(net_->layers()[0]);
- if (!md_layer) {
- throw std::runtime_error("set_input_arrays may only be called if the"
- " first layer is a MemoryDataLayer");
- }
-
- // check that we were passed appropriately-sized contiguous memory
- PyArrayObject* data_arr =
- reinterpret_cast<PyArrayObject*>(data_obj.ptr());
- PyArrayObject* labels_arr =
- reinterpret_cast<PyArrayObject*>(labels_obj.ptr());
- check_contiguous_array(data_arr, "data array", md_layer->channels(),
- md_layer->height(), md_layer->width());
- check_contiguous_array(labels_arr, "labels array", 1, 1, 1);
- if (PyArray_DIMS(data_arr)[0] != PyArray_DIMS(labels_arr)[0]) {
- throw std::runtime_error("data and labels must have the same first"
- " dimension");
- }
- if (PyArray_DIMS(data_arr)[0] % md_layer->batch_size() != 0) {
- throw std::runtime_error("first dimensions of input arrays must be a"
- " multiple of batch size");
- }
-
- // hold references
- input_data_ = data_obj;
- input_labels_ = labels_obj;
-
- md_layer->Reset(static_cast<float*>(PyArray_DATA(data_arr)),
- static_cast<float*>(PyArray_DATA(labels_arr)),
- PyArray_DIMS(data_arr)[0]);
-}
-
-PySGDSolver::PySGDSolver(const string& param_file) {
- // as in PyNet, (as a convenience, not a guarantee), create a Python
- // exception if param_file can't be opened
- CheckFile(param_file);
- solver_.reset(new SGDSolver<float>(param_file));
- // we need to explicitly store the net wrapper, rather than constructing
- // it on the fly, so that it can hold references to Python objects
- net_.reset(new PyNet(solver_->net()));
- for (int i = 0; i < solver_->test_nets().size(); ++i) {
- test_nets_.push_back(boost::make_shared<PyNet>(solver_->test_nets()[i]));
- }
-}
-
-void PySGDSolver::SolveResume(const string& resume_file) {
- CheckFile(resume_file);
- return solver_->Solve(resume_file);
-}
-
BOOST_PYTHON_MODULE(_caffe) {
- // Caffe utility methods
- bp::def("set_mode_cpu", &set_mode_cpu);
- bp::def("set_mode_gpu", &set_mode_gpu);
- bp::def("set_phase_train", &set_phase_train);
- bp::def("set_phase_test", &set_phase_test);
- bp::def("set_device", &Caffe::SetDevice);
-
- // below, we prepend an underscore to methods that will be replaced
- // in Python
- bp::class_<PyNet, shared_ptr<PyNet> >(
- "Net", bp::init<string, string>())
- .def(bp::init<string>())
- .def("copy_from", &PyNet::CopyTrainedLayersFrom)
- .def("share_with", &PyNet::ShareTrainedLayersWith)
- .def("_forward", &PyNet::Forward)
- .def("_backward", &PyNet::Backward)
- .def("reshape", &PyNet::Reshape)
- .add_property("_blobs", &PyNet::blobs)
- .add_property("layers", &PyNet::layers)
- .add_property("_blob_names", &PyNet::blob_names)
- .add_property("_layer_names", &PyNet::layer_names)
- .add_property("inputs", &PyNet::inputs)
- .add_property("outputs", &PyNet::outputs)
- .add_property("mean", &PyNet::mean_)
- .add_property("input_scale", &PyNet::input_scale_)
- .add_property("raw_scale", &PyNet::raw_scale_)
- .add_property("channel_swap", &PyNet::channel_swap_)
- .def("_set_input_arrays", &PyNet::set_input_arrays)
- .def("save", &PyNet::save);
-
- bp::class_<PyBlob<float>, PyBlobWrap>(
- "Blob", bp::no_init)
- .add_property("num", &PyBlob<float>::num)
- .add_property("channels", &PyBlob<float>::channels)
- .add_property("height", &PyBlob<float>::height)
- .add_property("width", &PyBlob<float>::width)
- .add_property("count", &PyBlob<float>::count)
- .def("reshape", &PyBlob<float>::Reshape)
- .add_property("data", &PyBlobWrap::get_data)
- .add_property("diff", &PyBlobWrap::get_diff);
-
- bp::class_<PyLayer>(
- "Layer", bp::no_init)
- .add_property("blobs", &PyLayer::blobs);
-
- bp::class_<PySGDSolver, boost::noncopyable>(
- "SGDSolver", bp::init<string>())
- .add_property("net", &PySGDSolver::net)
- .add_property("test_nets", &PySGDSolver::test_nets)
- .add_property("iter", &PySGDSolver::iter)
- .def("solve", &PySGDSolver::Solve)
- .def("solve", &PySGDSolver::SolveResume)
- .def("step", &PySGDSolver::Step);
-
- bp::class_<vector<shared_ptr<PyNet> > >("NetVec")
- .def(bp::vector_indexing_suite<vector<shared_ptr<PyNet> >, true>());
-
- bp::class_<vector<PyBlob<float> > >("BlobVec")
- .def(bp::vector_indexing_suite<vector<PyBlob<float> >, true>());
-
- bp::class_<vector<PyLayer> >("LayerVec")
- .def(bp::vector_indexing_suite<vector<PyLayer>, true>());
-
- bp::class_<vector<string> >("StringVec")
- .def(bp::vector_indexing_suite<vector<string> >());
-
import_array();
}
+++ /dev/null
-#ifndef PYTHON_CAFFE__CAFFE_HPP_
-#define PYTHON_CAFFE__CAFFE_HPP_
-
-#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
-
-#include <Python.h> // NOLINT(build/include_alpha)
-
-#include <boost/python.hpp>
-#include <boost/shared_ptr.hpp>
-#include <numpy/arrayobject.h>
-
-// these need to be included after boost on OS X
-#include <string> // NOLINT(build/include_order)
-#include <vector> // NOLINT(build/include_order)
-
-#include "caffe/caffe.hpp"
-
-namespace bp = boost::python;
-using boost::shared_ptr;
-
-namespace caffe {
-
-// Selecting mode and phase.
-void set_mode_cpu() { Caffe::set_mode(Caffe::CPU); }
-void set_mode_gpu() { Caffe::set_mode(Caffe::GPU); }
-void set_phase_train() { Caffe::set_phase(Caffe::TRAIN); }
-void set_phase_test() { Caffe::set_phase(Caffe::TEST); }
-
-// wrap shared_ptr<Blob> in a class that we construct in C++ and pass
-// to Python
-template <typename Dtype>
-class PyBlob {
- public:
- explicit PyBlob(const shared_ptr<Blob<Dtype> > &blob)
- : blob_(blob) {}
-
- int num() const { return blob_->num(); }
- int channels() const { return blob_->channels(); }
- int height() const { return blob_->height(); }
- int width() const { return blob_->width(); }
- int count() const { return blob_->count(); }
- void Reshape(const int n, const int c, const int h, const int w) {
- return blob_->Reshape(n, c, h, w);
- }
-
- // this is here only to satisfy boost's vector_indexing_suite
- bool operator == (const PyBlob &other) {
- return this->blob_ == other.blob_;
- }
-
- protected:
- shared_ptr<Blob<Dtype> > blob_;
-};
-
-// We need another wrapper (used as boost::python's HeldType) that receives a
-// self PyObject * which we can use as ndarray.base, so that data/diff memory
-// is not freed while still being used in Python.
-class PyBlobWrap : public PyBlob<float> {
- public:
- PyBlobWrap(PyObject *p, const PyBlob<float> &blob)
- : PyBlob<float>(blob), self_(p) {}
-
- bp::object get_data();
- bp::object get_diff();
-
- private:
- PyObject *self_;
-};
-
-class PyLayer {
- public:
- explicit PyLayer(const shared_ptr<Layer<float> > &layer)
- : layer_(layer) {}
-
- vector<PyBlob<float> > blobs() {
- return vector<PyBlob<float> >(layer_->blobs().begin(),
- layer_->blobs().end());
- }
-
- // this is here only to satisfy boost's vector_indexing_suite
- bool operator == (const PyLayer &other) {
- return this->layer_ == other.layer_;
- }
-
- protected:
- shared_ptr<Layer<float> > layer_;
-};
-
-class PyNet {
- public:
- // For cases where parameters will be determined later by the Python user,
- // create a Net with unallocated parameters (which will not be zero-filled
- // when accessed).
- explicit PyNet(string param_file) { Init(param_file); }
- PyNet(string param_file, string pretrained_param_file);
- explicit PyNet(shared_ptr<Net<float> > net)
- : net_(net) {}
- virtual ~PyNet() {}
-
- void Init(string param_file);
-
-
- // Generate Python exceptions for badly shaped or discontiguous arrays.
- inline void check_contiguous_array(PyArrayObject* arr, string name,
- int channels, int height, int width);
-
- void CopyTrainedLayersFrom(const string filename) {
- net_->CopyTrainedLayersFrom(filename);
- }
- void ShareTrainedLayersWith(PyNet* other) {
- net_->ShareTrainedLayersWith(other->net_.get());
- }
- void Forward(int start, int end) { net_->ForwardFromTo(start, end); }
- void Backward(int start, int end) { net_->BackwardFromTo(start, end); }
- void Reshape() { net_->Reshape(); }
-
- void set_input_arrays(bp::object data_obj, bp::object labels_obj);
-
- // Save the network weights to binary proto for net surgeries.
- void save(string filename) {
- NetParameter net_param;
- net_->ToProto(&net_param, false);
- WriteProtoToBinaryFile(net_param, filename.c_str());
- }
-
- vector<PyBlob<float> > blobs() {
- return vector<PyBlob<float> >(net_->blobs().begin(), net_->blobs().end());
- }
-
- vector<PyLayer> layers() {
- return vector<PyLayer>(net_->layers().begin(), net_->layers().end());
- }
-
- vector<string> blob_names() { return net_->blob_names(); }
- vector<string> layer_names() { return net_->layer_names(); }
-
- bp::list inputs() {
- bp::list input_blob_names;
- for (int i = 0; i < net_->input_blob_indices().size(); ++i) {
- input_blob_names.append(
- net_->blob_names()[net_->input_blob_indices()[i]]);
- }
- return input_blob_names;
- }
-
- bp::list outputs() {
- bp::list output_blob_names;
- for (int i = 0; i < net_->output_blob_indices().size(); ++i) {
- output_blob_names.append(
- net_->blob_names()[net_->output_blob_indices()[i]]);
- }
- return output_blob_names;
- }
-
- // Input preprocessing configuration attributes. These are public for
- // direct access from Python.
- bp::dict mean_;
- bp::dict input_scale_;
- bp::dict raw_scale_;
- bp::dict channel_swap_;
-
- // this is here only to satisfy boost's vector_indexing_suite
- bool operator == (const PyNet &other) {
- return this->net_ == other.net_;
- }
-
- protected:
- // The pointer to the internal caffe::Net instance.
- shared_ptr<Net<float> > net_;
- // if taking input from an ndarray, we need to hold references
- bp::object input_data_;
- bp::object input_labels_;
-};
-
-class PySGDSolver {
- public:
- explicit PySGDSolver(const string& param_file);
-
- shared_ptr<PyNet> net() { return net_; }
- vector<shared_ptr<PyNet> > test_nets() { return test_nets_; }
- int iter() { return solver_->iter(); }
- void Solve() { return solver_->Solve(); }
- void Step(int iters) { solver_->Step(iters); }
- void SolveResume(const string& resume_file);
-
- protected:
- shared_ptr<PyNet> net_;
- vector<shared_ptr<PyNet> > test_nets_;
- shared_ptr<SGDSolver<float> > solver_;
-};
-
-// Declare the module init function created by boost::python, so that we can
-// use this module from C++ when embedding Python.
-PyMODINIT_FUNC init_caffe(void);
-
-} // namespace caffe
-
-#endif
projects / platform / upstream / caffe.git / commitdiff