lots of corrections to dependencies etc., things seem to mostly build

coherently now

diff --git a/Makefile b/Makefile
index ad74fb8..26c9517 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-# The makefile for caffe. Extremely hacky.
+# The makefile for caffe. Pretty hacky.
 PROJECT := caffe
 
 include Makefile.config
@@ -33,7 +33,7 @@ TOOL_SRCS := $(shell find tools -name "*.cpp")
 EXAMPLE_SRCS := $(shell find examples -name "*.cpp")
 # BUILD_INCLUDE_DIR contains any generated header files we want to include.
 BUILD_INCLUDE_DIR := $(BUILD_DIR)/include
-# PROTO_SRCS are the protocol buffer definitions
+# PROTO_SRCS are the protocol buffer defions
 PROTO_SRC_DIR := src/$(PROJECT)/proto
 PROTO_SRCS := $(wildcard $(PROTO_SRC_DIR)/*.proto)
 # PROTO_BUILD_DIR will contain the .cc and obj files generated from
@@ -63,32 +63,45 @@ MAT$(PROJECT)_SO := matlab/$(PROJECT)/$(PROJECT)
 # Derive generated files
 ##############################
 # The generated files for protocol buffers
+PROTO_GEN_HEADER_SRCS := $(addprefix $(PROTO_BUILD_DIR)/, \
+       $(notdir ${PROTO_SRCS:.proto=.pb.h}))
 PROTO_GEN_HEADER := $(addprefix $(PROTO_BUILD_INCLUDE_DIR)/, \
        $(notdir ${PROTO_SRCS:.proto=.pb.h}))
 HXX_SRCS += $(PROTO_GEN_HEADER)
 PROTO_GEN_CC := $(addprefix $(BUILD_DIR)/, ${PROTO_SRCS:.proto=.pb.cc})
-PROTO_GEN_PY := ${PROTO_SRCS:.proto=_pb2.py}
+PROTO_GEN_PY := $(foreach file,${PROTO_SRCS:.proto=_pb2.py},python/$(PROJECT)/proto/$(notdir $(file)))
 # The objects corresponding to the source files
 # These objects will be linked into the final shared library, so we
 # exclude the tool, example, and test objects.
 CXX_OBJS := $(addprefix $(BUILD_DIR)/, ${CXX_SRCS:.cpp=.o})
 CU_OBJS := $(addprefix $(BUILD_DIR)/, ${CU_SRCS:.cu=.cuo})
 PROTO_OBJS := ${PROTO_GEN_CC:.cc=.o}
+OBJ_BUILD_DIR := $(BUILD_DIR)/src/$(PROJECT)
+LAYER_BUILD_DIR := $(OBJ_BUILD_DIR)/layers
+UTIL_BUILD_DIR := $(OBJ_BUILD_DIR)/util
 OBJS := $(PROTO_OBJS) $(CXX_OBJS) $(CU_OBJS)
 # tool, example, and test objects
 TOOL_OBJS := $(addprefix $(BUILD_DIR)/, ${TOOL_SRCS:.cpp=.o})
-EXAMPLE_OBJS := $(addprefix $(BUILD_DIR)/, ${EXAMPLE_SRCS:.cpp=.o})
+TOOL_BUILD_DIR := $(BUILD_DIR)/tools
+TOOL_BUILD_DIRS := $(sort $(foreach obj,$(TOOL_OBJS),$(dir $(obj))))
+TEST_BUILD_DIR := $(BUILD_DIR)/src/$(PROJECT)/test
 TEST_OBJS := $(addprefix $(BUILD_DIR)/, ${TEST_SRCS:.cpp=.o})
 GTEST_OBJ := $(addprefix $(BUILD_DIR)/, ${GTEST_SRC:.cpp=.o})
+GTEST_BUILD_DIR := $(dir $(GTEST_OBJ))
+EXAMPLE_OBJS := $(addprefix $(BUILD_DIR)/, ${EXAMPLE_SRCS:.cpp=.o})
+EXAMPLE_BUILD_DIR := $(BUILD_DIR)/examples
+EXAMPLE_BUILD_DIRS := $(EXAMPLE_BUILD_DIR)
+EXAMPLE_BUILD_DIRS += $(foreach obj,$(EXAMPLE_OBJS),$(dir $(obj)))
 # tool, example, and test bins
 TOOL_BINS := ${TOOL_OBJS:.o=.bin}
 EXAMPLE_BINS := ${EXAMPLE_OBJS:.o=.bin}
 TEST_BINS := ${TEST_OBJS:.o=.testbin}
-TEST_BUILD_SUB_DIR := src/$(PROJECT)/test
-TEST_DIR = $(BUILD_DIR)/$(TEST_BUILD_SUB_DIR)
-TEST_ALL_BIN := $(TEST_DIR)/test_all.testbin
+TEST_ALL_BIN := $(TEST_BUILD_DIR)/test_all.testbin
+TEST_ALL_BINS := $(TEST_ALL_BIN) $(TEST_BINS)
 # A shortcut to the directory of test binaries for convenience.
-TEST_DIR_LINK := $(BUILD_DIR)/test
+TEST_LINK_DIR := $(BUILD_DIR)/test
+TEST_ALL_BIN_LINKS := $(foreach \
+               bin,$(TEST_ALL_BINS),$(TEST_LINK_DIR)/$(notdir $(bin)))
 
 ##############################
 # Derive include and lib directories
@@ -110,7 +123,27 @@ LIBRARIES := cudart cublas curand \
 PYTHON_LIBRARIES := boost_python python2.7
 WARNINGS := -Wall
 
-ifdef DEBUG
+##############################
+# Set build directories
+##############################
+
+DISTRIBUTE_SUBDIRS := $(DISTRIBUTE_DIR)/bin $(DISTRIBUTE_DIR)/lib
+DIST_ALIASES := dist
+ifneq ($(strip $(DISTRIBUTE_DIR)),distribute)
+               DIST_ALIASES += distribute
+endif
+
+ALL_BUILD_DIRS := $(BUILD_DIR) $(OBJ_BUILD_DIR) \
+               $(LAYER_BUILD_DIR) $(UTIL_BUILD_DIR) $(TOOL_BUILD_DIRS) \
+               $(TEST_BUILD_DIR) $(TEST_LINK_DIR) $(GTEST_BUILD_DIR) \
+               $(EXAMPLE_BUILD_DIRS) \
+               $(PROTO_BUILD_DIR) $(PROTO_BUILD_INCLUDE_DIR) \
+               $(DISTRIBUTE_SUBDIRS)
+
+ALL_BUILD_DIRS := $(sort $(ALL_BUILD_DIRS))
+
+DEBUG ?= 0
+ifeq ($(DEBUG), 1)
        COMMON_FLAGS := -DDEBUG -g -O0
 else
        COMMON_FLAGS := -DNDEBUG -O2
@@ -134,31 +167,34 @@ LDFLAGS += $(foreach librarydir,$(LIBRARY_DIRS),-L$(librarydir)) \
                $(foreach library,$(LIBRARIES),-l$(library))
 PYTHON_LDFLAGS := $(LDFLAGS) $(foreach library,$(PYTHON_LIBRARIES),-l$(library))
 
+# 'superclean' target recursively* deletes all files ending with an extension
+# suggesting that Caffe built them.  This may be useful if you've built older
+# versions of Caffe that do not place all generated files in a location known
+# to make clean.
+#
+# 'supercleanlist' will list the files to be deleted by make superclean.
+#
+# * Recursive with the exception that symbolic links are never followed, per the
+# default behavior of 'find'.
+SUPERCLEAN_EXTS := .so .a .o .bin .testbin .pb.cc .pb.h _pb2.py .cuo
+
 ##############################
 # Define build targets
 ##############################
-.PHONY: all init test clean linecount lint tools examples distribute \
+.PHONY: all test clean linecount lint tools examples dist \
        py mat py$(PROJECT) mat$(PROJECT) proto runtest \
-       superclean supercleanlist supercleanfiles \
-       testshortcut
+       superclean supercleanlist supercleanfiles
 
-all: init $(NAME) $(STATIC_NAME) tools examples
-       @echo $(CXX_OBJS)
+.SECONDARY: $(PROTO_GEN_HEADER_SRCS) $(TEST_BINS)
 
-init:
-       @ mkdir -p $(foreach obj,$(OBJS),$(dir $(obj)))
-       @ mkdir -p $(foreach obj,$(TOOL_OBJS),$(dir $(obj)))
-       @ mkdir -p $(foreach obj,$(EXAMPLE_OBJS),$(dir $(obj)))
-       @ mkdir -p $(foreach obj,$(TEST_OBJS),$(dir $(obj)))
-       @ mkdir -p $(foreach obj,$(GTEST_OBJ),$(dir $(obj)))
+all: $(NAME) $(STATIC_NAME) tools examples
 
 linecount: clean
        cloc --read-lang-def=$(PROJECT).cloc src/$(PROJECT)/
 
 lint: $(LINT_REPORT)
 
-$(LINT_REPORT): $(NONGEN_CXX_SRCS)
-       @ mkdir -p $(BUILD_DIR)
+$(LINT_REPORT): $(NONGEN_CXX_SRCS) | $(BUILD_DIR)
        @ (python ./scripts/cpp_lint.py $(NONGEN_CXX_SRCS) > $(LINT_REPORT) 2>&1 \
                && (rm -f $(FAILED_LINT_REPORT); echo "No lint errors!")) || ( \
                        mv $(LINT_REPORT) $(FAILED_LINT_REPORT); \
@@ -166,133 +202,119 @@ $(LINT_REPORT): $(NONGEN_CXX_SRCS)
                        echo "Found 1 or more lint errors; see log at $(FAILED_LINT_REPORT)"; \
                        exit 1)
 
-test: init $(TEST_BINS) $(TEST_ALL_BIN)
+test: $(TEST_ALL_BIN_LINKS)
 
-tools: init proto $(TOOL_BINS)
+tools: $(TOOL_BINS)
 
-examples: init $(EXAMPLE_BINS)
+examples: $(EXAMPLE_BINS)
 
 py$(PROJECT): py
 
-py: init $(STATIC_NAME) $(PY$(PROJECT)_SRC) $(PROTO_GEN_PY)
-       $(CXX) -shared -o $(PY$(PROJECT)_SO) $(PY$(PROJECT)_SRC) \
+py: $(PY$(PROJECT)_SO) $(PROTO_GEN_PY)
+
+$(PY$(PROJECT)_SO): $(STATIC_NAME) $(PY$(PROJECT)_SRC)
+       $(CXX) -shared -o $@ $(PY$(PROJECT)_SRC) \
                $(STATIC_NAME) $(CXXFLAGS) $(PYTHON_LDFLAGS)
-       @echo
 
 mat$(PROJECT): mat
 
-mat: init $(STATIC_NAME) $(MAT$(PROJECT)_SRC)
+mat: $(STATIC_NAME) $(MAT$(PROJECT)_SRC)
        $(MATLAB_DIR)/bin/mex $(MAT$(PROJECT)_SRC) $(STATIC_NAME) \
                CXXFLAGS="\$$CXXFLAGS $(CXXFLAGS) $(WARNINGS)" \
                CXXLIBS="\$$CXXLIBS $(LDFLAGS)" \
                -o $(MAT$(PROJECT)_SO)
-       @echo
+       @ echo
+
+runtest: $(TEST_ALL_BIN)
+       $(TEST_ALL_BIN) $(TEST_GPUID)
+
+$(ALL_BUILD_DIRS):
+       @ mkdir -p $@
 
-$(NAME): init $(PROTO_OBJS) $(OBJS)
+$(NAME): $(PROTO_OBJS) $(OBJS)
        $(CXX) -shared -o $(NAME) $(OBJS) $(CXXFLAGS) $(LDFLAGS) $(WARNINGS)
-       @echo
 
-$(STATIC_NAME): init $(PROTO_OBJS) $(OBJS)
+$(STATIC_NAME): $(PROTO_OBJS) $(OBJS)
        ar rcs $(STATIC_NAME) $(PROTO_OBJS) $(OBJS)
-       @echo
 
-runtest: $(TEST_ALL_BIN)
-       $(TEST_ALL_BIN) $(TEST_GPUID)
-
-$(BUILD_DIR)/src/$(PROJECT)/test/%.testbin: \
-               $(BUILD_DIR)/src/$(PROJECT)/test/%.o \
-               $(GTEST_OBJ) $(STATIC_NAME) testshortcut
+$(TEST_BUILD_DIR)/%.testbin: $(TEST_BUILD_DIR)/%.o $(GTEST_OBJ) $(STATIC_NAME) \
+               | $(TEST_BUILD_DIR)
        $(CXX) $(TEST_MAIN_SRC) $< $(GTEST_OBJ) $(STATIC_NAME) \
                -o $@ $(CXXFLAGS) $(LDFLAGS) $(WARNINGS)
 
-$(TEST_ALL_BIN): $(GTEST_OBJ) $(STATIC_NAME) $(TEST_OBJS) testshortcut
+$(TEST_ALL_BIN): $(TEST_MAIN_SRC) $(TEST_OBJS) $(GTEST_OBJ) $(STATIC_NAME)
        $(CXX) $(TEST_MAIN_SRC) $(TEST_OBJS) $(GTEST_OBJ) $(STATIC_NAME) \
                -o $(TEST_ALL_BIN) $(CXXFLAGS) $(LDFLAGS) $(WARNINGS)
 
-testshortcut: | $(TEST_DIR_LINK)
-
-$(TEST_DIR_LINK): | $(TEST_DIR)
-       ln -s $(TEST_BUILD_SUB_DIR) $(TEST_DIR_LINK)
-
-$(TEST_DIR):
-       mkdir -p $(TEST_DIR)
+$(TEST_LINK_DIR)/%.testbin: $(TEST_BUILD_DIR)/%.testbin | $(TEST_LINK_DIR)
+       @ $(RM) $@
+       @ ln -s ../../$(TEST_BUILD_DIR)/$(@F) $@
 
 $(TOOL_BINS): %.bin : %.o $(STATIC_NAME)
        $(CXX) $< $(STATIC_NAME) -o $@ $(CXXFLAGS) $(LDFLAGS) $(WARNINGS)
-       @echo
+       @ echo
 
 $(EXAMPLE_BINS): %.bin : %.o $(STATIC_NAME)
        $(CXX) $< $(STATIC_NAME) -o $@ $(CXXFLAGS) $(LDFLAGS) $(WARNINGS)
-       @echo
-
-$(BUILD_DIR)/src/$(PROJECT)/%.o: src/$(PROJECT)/%.cpp
-       $(CXX) $< $(CXXFLAGS) -c -o $@
-       @echo
-
-$(OBJS): $(PROTO_GEN_HEADER) $(HXX_SRCS)
-       @echo matched the first objs!!
-
+       @ echo
 
-LAYERS_DIR := $(BUILD_DIR)/src/$(PROJECT)/layers
-$(LAYERS_DIR):
-       @ mkdir -p $(LAYERS_DIR)
-
-$(BUILD_DIR)/src/$(PROJECT)/layers/%.o: \
-               src/$(PROJECT)/layers/%.cpp $(HXX_SRCS) | $(LAYERS_DIR)
+$(LAYER_BUILD_DIR)/%.o: \
+               src/$(PROJECT)/layers/%.cpp $(HXX_SRCS) | $(LAYER_BUILD_DIR)
        $(CXX) $< $(CXXFLAGS) -c -o $@
-       @echo
 
-$(BUILD_DIR)/src/$(PROJECT)/proto/%.o: src/$(PROJECT)/proto/%.cc src/$(PROJECT)/proto/%.h
+$(PROTO_BUILD_DIR)/%.pb.o: $(PROTO_BUILD_DIR)/%.pb.cc \
+               $(PROTO_GEN_HEADER) | $(PROTO_BUILD_DIR)
        $(CXX) $< $(CXXFLAGS) -c -o $@
-       @echo
+       @ echo
 
-$(BUILD_DIR)/src/$(PROJECT)/test/%.o: $(PROTO_GEN_HEADER) src/$(PROJECT)/test/%.cpp
+$(TEST_BUILD_DIR)/%.o: src/$(PROJECT)/test/%.cpp $(HXX_SRCS) | $(TEST_BUILD_DIR)
        $(CXX) $< $(CXXFLAGS) -c -o $@
 
-$(BUILD_DIR)/src/$(PROJECT)/util/%.o: src/$(PROJECT)/util/%.cpp
+$(UTIL_BUILD_DIR)/%.o: src/$(PROJECT)/util/%.cpp $(HXX_SRCS) | $(UTIL_BUILD_DIR)
        $(CXX) $< $(CXXFLAGS) -c -o $@
-       @echo
+       @ echo
 
-$(BUILD_DIR)/src/gtest/%.o: src/gtest/%.cpp
+$(GTEST_OBJ): $(GTEST_SRC) | $(GTEST_BUILD_DIR)
        $(CXX) $< $(CXXFLAGS) -c -o $@
-       @echo
+       @ echo
 
-$(BUILD_DIR)/src/$(PROJECT)/layers/%.cuo: src/$(PROJECT)/layers/%.cu
+$(LAYER_BUILD_DIR)/%.cuo: \
+               src/$(PROJECT)/layers/%.cu $(HXX_SRCS) | $(LAYER_BUILD_DIR)
        $(CUDA_DIR)/bin/nvcc $(NVCCFLAGS) $(CUDA_ARCH) -c $< -o $@
-       @echo
+       @ echo
 
-$(BUILD_DIR)/src/$(PROJECT)/util/%.cuo: src/$(PROJECT)/util/%.cu
+$(UTIL_BUILD_DIR)/%.cuo: src/$(PROJECT)/util/%.cu | $(UTIL_BUILD_DIR)
        $(CUDA_DIR)/bin/nvcc $(NVCCFLAGS) $(CUDA_ARCH) -c $< -o $@
-       @echo
+       @ echo
 
-$(BUILD_DIR)/tools/%.o: tools/%.cpp $(PROTO_GEN_HEADER)
+$(TOOL_BUILD_DIR)/%.o: tools/%.cpp $(PROTO_GEN_HEADER) | $(TOOL_BUILD_DIR)
        $(CXX) $< $(CXXFLAGS) -c -o $@ $(LDFLAGS)
-       @echo
+       @ echo
 
-$(BUILD_DIR)/examples/%.o: examples/%.cpp $(PROTO_GEN_HEADER)
+$(EXAMPLE_BUILD_DIR)/%.o: examples/%.cpp $(PROTO_GEN_HEADER) \
+               | $(EXAMPLE_BUILD_DIRS)
        $(CXX) $< $(CXXFLAGS) -c -o $@ $(LDFLAGS)
-       @echo
+       @ echo
+
+$(BUILD_DIR)/src/$(PROJECT)/%.o: src/$(PROJECT)/%.cpp $(HXX_SRCS)
+       $(CXX) $< $(CXXFLAGS) -c -o $@
+       @ echo
 
 $(PROTO_GEN_PY): $(PROTO_SRCS)
        protoc --proto_path=src --python_out=python $(PROTO_SRCS)
-       @echo
+       @ echo
 
-proto: init $(PROTO_GEN_CC) $(PROTO_GEN_HEADER)
-       @echo PROTO_GEN_CC: $(PROTO_GEN_CC)
-       @echo PROTO_GEN_HEADER: $(PROTO_GEN_HEADER)
-       @echo PROTO_OBJS: $(PROTO_OBJS)
+proto: $(PROTO_GEN_CC) $(PROTO_GEN_HEADER)
 
-$(PROTO_BUILD_DIR)/%.pb.cc $(PROTO_BUILD_DIR)/%.pb.h \
-               $(PROTO_BUILD_INCLUDE_DIR)/%.pb.h: \
-               $(PROTO_SRC_DIR)/%.proto | $(PROTO_BUILD_DIR) $(PROTO_BUILD_INCLUDE_DIR)
+$(PROTO_BUILD_DIR)/%.pb.cc $(PROTO_BUILD_DIR)/%.pb.h : \
+               $(PROTO_SRC_DIR)/%.proto | $(PROTO_BUILD_DIR)
        protoc --proto_path=src --cpp_out=build/src $<
-       cp $(PROTO_BUILD_DIR)/$(*F).pb.h $(PROTO_BUILD_INCLUDE_DIR)/$(*F).pb.h
 
-$(PROTO_BUILD_DIR):
-       mkdir -p $(PROTO_BUILD_DIR)
-
-$(PROTO_BUILD_INCLUDE_DIR):
-       mkdir -p $(PROTO_BUILD_INCLUDE_DIR)
+$(PROTO_BUILD_INCLUDE_DIR)/%.pb.h: $(PROTO_BUILD_DIR)/%.pb.h \
+               | $(PROTO_BUILD_INCLUDE_DIR)
+       @ $(RM) $(PROTO_BUILD_INCLUDE_DIR)/$(*F).pb.h
+       @ ln -s ../../../../$(PROTO_BUILD_DIR)/$(*F).pb.h \
+                       $(PROTO_BUILD_INCLUDE_DIR)/$(*F).pb.h
 
 clean:
        @- $(RM) $(NAME) $(STATIC_NAME)
@@ -303,20 +325,9 @@ clean:
        @- $(RM) -rf $(BUILD_DIR)
        @- $(RM) -rf $(DISTRIBUTE_DIR)
 
-# make superclean recursively* deletes all files ending with an extension
-# suggesting that Caffe built them.  This may be useful if you've built older
-# versions of Caffe that do not place all generated files in a location known
-# to make clean.
-#
-# make supercleanlist will list the files to be deleted by make superclean.
-#
-# * Recursive with the exception that symbolic links are never followed, per the
-# default behavior of 'find'.
-SUPERCLEAN_EXTS := .so .a .o .bin .testbin .pb.cc .pb.h _pb2.py .cuo
-
 supercleanfiles:
        $(eval SUPERCLEAN_FILES := $(strip \
-               $(foreach ext,$(SUPERCLEAN_EXTS), $(shell find . -name '*$(ext)'))))
+                       $(foreach ext,$(SUPERCLEAN_EXTS), $(shell find . -name '*$(ext)'))))
 
 supercleanlist: supercleanfiles
        @ \
@@ -336,16 +347,15 @@ superclean: clean supercleanfiles
          $(RM) $(SUPERCLEAN_FILES); \
        fi
 
-distribute: all
-       mkdir $(DISTRIBUTE_DIR)
+$(DIST_ALIASES): $(DISTRIBUTE_DIR)
+
+$(DISTRIBUTE_DIR): all py $(HXX_SRCS) | $(DISTRIBUTE_SUBDIRS)
        # add include
        cp -r include $(DISTRIBUTE_DIR)/
        # add tool and example binaries
-       mkdir $(DISTRIBUTE_DIR)/bin
        cp $(TOOL_BINS) $(DISTRIBUTE_DIR)/bin
        cp $(EXAMPLE_BINS) $(DISTRIBUTE_DIR)/bin
        # add libraries
-       mkdir $(DISTRIBUTE_DIR)/lib
        cp $(NAME) $(DISTRIBUTE_DIR)/lib
        cp $(STATIC_NAME) $(DISTRIBUTE_DIR)/lib
        # add python - it's not the standard way, indeed...