diff --git a/TPL/libsvm-3.1-custom/Makefile b/TPL/libsvm-3.1-custom/Makefile index 9038d32cd..ac6f251b5 100644 --- a/TPL/libsvm-3.1-custom/Makefile +++ b/TPL/libsvm-3.1-custom/Makefile @@ -1,12 +1,22 @@ CXX ?= g++ CFLAGS = -Wall -Wconversion -O3 -fPIC SHVER = 2 +PYTHON = python2.7 +.PHONY = test all: svm-train svm-predict svm-scale svm-prob binary-train binary-predict lib: svm.o $(CXX) -shared -dynamiclib svm.o -o libsvm.so.$(SHVER) +svm-test: svm-test.o svm.o + $(CXX) svm-test.o svm.o -lgtest -lpthread -o svm-test + +test: svm-test lib + mkdir -p ./python/lib && cp libsvm.so.$(SHVER) python/lib/ + valgrind --leak-check=full ./svm-test + cd python && $(PYTHON) svm_test.py + svm-predict: svm-predict.c svm.o $(CXX) $(CFLAGS) svm-predict.c svm.o -o svm-predict -lm svm-train: svm-train.c svm.o @@ -21,6 +31,9 @@ svm.o: svm.cpp svm.h eval.o: eval.cpp eval.h svm.h $(CXX) $(CFLAGS) -c eval.cpp -o eval.o +svm-test.o: svm-test.cpp svm.o + $(CXX) -std=c++11 $(CFLAGS) -c svm-test.cpp + binary-predict: binary-predict.c eval.o svm.o $(CXX) $(CFLAGS) binary-predict.c eval.o svm.o -o binary-predict -lm binary-train: binary-train.c eval.o svm.o @@ -28,4 +41,6 @@ binary-train: binary-train.c eval.o svm.o clean: - rm -f *~ svm.o eval.o svm-train svm-predict svm-scale svm-prob binary-train binary-predict libsvm.so.$(SHVER) + rm -f *~ svm.o svm-test.o eval.o svm-train svm-predict svm-scale \ + svm-prob binary-train binary-predict svm-test libsvm.so.$(SHVER) + rm -R ./python/lib diff --git a/TPL/libsvm-3.1-custom/python/svm.py b/TPL/libsvm-3.1-custom/python/svm.py index 771e65720..feb239b03 100644 --- a/TPL/libsvm-3.1-custom/python/svm.py +++ b/TPL/libsvm-3.1-custom/python/svm.py @@ -321,7 +321,8 @@ def toPyModel(model_ptr): fillprototype(libsvm.svm_save_model, c_int, [c_char_p, POINTER(svm_model)]) fillprototype(libsvm.svm_load_model, POINTER(svm_model), [c_char_p]) -fillprototype(libsvm.svm_load_model_from_bytes, POINTER(svm_model), [POINTER(c_ubyte), c_int]) +fillprototype(libsvm.load_model_from_bytes, POINTER(svm_model), [POINTER(c_ubyte), c_int]) +fillprototype(libsvm.convert_model_to_bytes, None, [POINTER(svm_model), POINTER(POINTER(c_ubyte)), POINTER(c_size_t)]) fillprototype(libsvm.svm_get_svm_type, c_int, [POINTER(svm_model)]) fillprototype(libsvm.svm_get_nr_class, c_int, [POINTER(svm_model)]) diff --git a/TPL/libsvm-3.1-custom/python/svm_test.py b/TPL/libsvm-3.1-custom/python/svm_test.py new file mode 100644 index 000000000..95fc733d7 --- /dev/null +++ b/TPL/libsvm-3.1-custom/python/svm_test.py @@ -0,0 +1,35 @@ +from svmutil import * +import unittest +import filecmp + + +class TestSVM(unittest.TestCase): + + model_names = [ + "../data/svmguide1.model", + "../data/svmguide1.scale.model", + ] + + def save_model_and_check(self, model_true, model_false): + model = svm_load_model(model_true) + bytes_list = svm_conv_model_to_bytes(model) + tmp_file_name = "/tmp/tmp.model" + svm_save_model(tmp_file_name, svm_load_model_from_bytes(bytes_list)) + assert(filecmp.cmp(tmp_file_name, model_true) is True) + assert(filecmp.cmp(tmp_file_name, model_false) is False) + + def save_model_and_check_bytes_file_name(self, model_true, model_false): + model = svm_load_model(model_true) + bytes_list = svm_conv_model_to_bytes(model) + tmp_file_name = '/tmp/tmp.model'.encode("utf-8") + svm_save_model(tmp_file_name, svm_load_model_from_bytes(bytes_list)) + assert(filecmp.cmp(tmp_file_name, model_true) is True) + assert(filecmp.cmp(tmp_file_name, model_false) is False) + + def test_save_model(self): + self.save_model_and_check(self.model_names[0], self.model_names[1]) + self.save_model_and_check(self.model_names[1], self.model_names[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/TPL/libsvm-3.1-custom/python/svmutil.py b/TPL/libsvm-3.1-custom/python/svmutil.py index dee286f6c..9724b1e2b 100644 --- a/TPL/libsvm-3.1-custom/python/svmutil.py +++ b/TPL/libsvm-3.1-custom/python/svmutil.py @@ -2,8 +2,10 @@ from __future__ import print_function from svm import * +import numpy as np import sys +import six def svm_read_problem(data_file_name): """ @@ -33,31 +35,49 @@ def svm_load_model(model_file_name): Load a LIBSVM model from model_file_name and return. """ - model = libsvm.svm_load_model(model_file_name) + # Insconsistency between Python 3 and Python 2. + if isinstance(model_file_name, six.binary_type): + model_file_name_bytes = model_file_name + else: + model_file_name_bytes = six.b(model_file_name) + + model = libsvm.svm_load_model(model_file_name_bytes) if not model: print("can't open model file %s" % model_file_name) return None model = toPyModel(model) return model -def svm_load_model_from_bytes(bytes): - model = libsvm.svm_load_model_from_bytes(bytes, len(bytes)) +def svm_load_model_from_bytes(bytes_list): + bytes_list_pointer = bytes_list.ctypes.data_as(POINTER(c_ubyte)) + model = libsvm.load_model_from_bytes(bytes_list_pointer, len(bytes_list)) if not model: print("can't load model from the bytes") return None model = toPyModel(model) return model +def svm_conv_model_to_bytes(model): + bytes_list = POINTER(c_ubyte)() + bytes_list_len = c_size_t() + libsvm.convert_model_to_bytes(model, byref(bytes_list), byref(bytes_list_len)) + array_type = c_ubyte * bytes_list_len.value + array_pointer = cast(bytes_list, POINTER(array_type)) + py_bytes_list = np.frombuffer(array_pointer.contents, dtype=np.uint8) + return py_bytes_list + def svm_save_model(model_file_name, model): """ svm_save_model(model_file_name, model) -> None Save a LIBSVM model to the file model_file_name. """ - if sys.version_info >= (3, 0): - libsvm.svm_save_model(bytes(model_file_name, encoding='utf-8'), model) + # Insconsistency between Python 3 and Python 2. + if isinstance(model_file_name, six.binary_type): + model_file_name_bytes = model_file_name else: - libsvm.svm_save_model(model_file_name, model) + model_file_name_bytes = six.b(model_file_name) + libsvm.svm_save_model(model_file_name_bytes, model) def evaluations(ty, pv): """ diff --git a/TPL/libsvm-3.1-custom/svm-test.cpp b/TPL/libsvm-3.1-custom/svm-test.cpp new file mode 100644 index 000000000..d47e6d8c7 --- /dev/null +++ b/TPL/libsvm-3.1-custom/svm-test.cpp @@ -0,0 +1,94 @@ +// +// Created by chaturvedi on 2/20/18. +// + +#include +#include "svm.h" +#include + + +class SVMTest : public ::testing::Test +{ + protected: + svm_model *sample_model1; + svm_model *sample_model2; + + virtual void SetUp() { + sample_model1 = svm_load_model("./data/svmguide1.model"); + sample_model2 = svm_load_model("./data/svmguide1.scale.model"); + } + virtual void TearDown(){ + svm_free_and_destroy_model(&sample_model1); + svm_free_and_destroy_model(&sample_model2); + } + + public: + // TODO (Mmanu) : Suboptimal because we're saving a file and diff-ing it + // But letting it be because we're just testing + bool check_models_equal(svm_model* m, const char* orig_model_file_name){ + const char *tmp_file_name = "/tmp/tmp_svm.model"; + const char *diff_file = "/tmp/tmp_diff.op"; + + svm_save_model(tmp_file_name, m); + char buff[100]; + // Use diff to make sure that the models written are exactly the same + sprintf(buff, "diff %s %s > %s", tmp_file_name, orig_model_file_name, diff_file); + system(buff); + + // Check that the diff output is an empty file + FILE *fp = fopen(diff_file, "r"); + // Got to the 0th place before end. + fseek(fp, 0, SEEK_END); + long size = ftell(fp); + bool res = false; + + fclose(fp); + if(size == 0) res = true; + std::remove(diff_file); + std::remove(tmp_file_name); + return res; + } + + void test_model(const char *original_model_name, const char *other_model_name){ + svm_model* original_model = svm_load_model(original_model_name); + + + unsigned char* p; + unsigned long size; + // Load and check the model from bytes array + convert_model_to_bytes(original_model, p, size); + svm_model* returned_model = load_model_from_bytes(p, size); + ASSERT_TRUE(check_models_equal(returned_model, original_model_name)); + ASSERT_FALSE(check_models_equal(returned_model, other_model_name)); + free(p); + svm_free_and_destroy_model(&returned_model); + + // Load and check the model from bytes vector + std::vector* p_vec = convert_model_to_bytes_vector(original_model); + svm_model* returned_model_vector = load_model_from_bytes_vector(p_vec); + ASSERT_TRUE(check_models_equal(returned_model_vector, original_model_name)); + ASSERT_FALSE(check_models_equal(returned_model_vector, other_model_name)); + delete p_vec; + svm_free_and_destroy_model(&returned_model_vector); + svm_free_and_destroy_model(&original_model); + } + +}; + +TEST_F(SVMTest, checkFirstModel) { + const char* testing_model_name = "./data/svmguide1.model"; + const char* other_model_name = "./data/svmguide1.scale.model"; + test_model(testing_model_name, other_model_name); +} + +TEST_F(SVMTest, checkSecondModel) { + const char* testing_model_name = "./data/svmguide1.scale.model"; + const char* other_model_name = "./data/svmguide1.model"; + test_model(testing_model_name, other_model_name); +} + +int main(int argc, char** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/TPL/libsvm-3.1-custom/svm.cpp b/TPL/libsvm-3.1-custom/svm.cpp index d32326bca..aba7c7e77 100644 --- a/TPL/libsvm-3.1-custom/svm.cpp +++ b/TPL/libsvm-3.1-custom/svm.cpp @@ -2729,11 +2729,8 @@ static const char *kernel_type_table[]= "linear","polynomial","rbf","sigmoid","precomputed", "histogram", "NMI", NULL }; -int svm_save_model(const char *model_file_name, const svm_model *model) +int save_model_to_file_pointer(FILE *fp, const svm_model *model) { - FILE *fp = fopen(model_file_name,"w"); - if(fp==NULL) return -1; - const svm_parameter& param = model->param; fprintf(fp,"svm_type %s\n", svm_type_table[param.svm_type]); @@ -2817,6 +2814,31 @@ int svm_save_model(const char *model_file_name, const svm_model *model) else return 0; } +int svm_save_model(const char *model_file_name, const svm_model *model){ + FILE *fp = fopen(model_file_name, "w"); + if(fp==NULL) return -1; + return save_model_to_file_pointer(fp, model); +} + +void convert_model_to_bytes(svm_model* model, unsigned char* &buffer, + size_t &size) { + char *buf1 = NULL; + FILE *fp = open_memstream(&buf1, &size); + save_model_to_file_pointer(fp, model); + buffer = (unsigned char *) buf1; +} + +std::vector* convert_model_to_bytes_vector(svm_model* model){ + size_t size; + unsigned char* buffer; + convert_model_to_bytes(model, buffer, size); + std::vector* A = new std::vector(size); + A -> assign(buffer, buffer + size); + free(buffer); + + return A; +} + static char *line = NULL; static int max_line_len; @@ -2838,8 +2860,7 @@ static char* readline(FILE *input) return line; } -// TODO May need to remove this. Ask Matt. -svm_model *svm_load_model_from_bytes_vector(std::vector& model_bytes) { +svm_model *load_model_from_bytes_vector(std::vector& model_bytes) { const char* tmp_file_name = "tmp_svm_cpp.model"; std::ofstream out(tmp_file_name, std::ios::binary); out.write(reinterpret_cast(model_bytes.data()), model_bytes.size()); @@ -2848,7 +2869,7 @@ svm_model *svm_load_model_from_bytes_vector(std::vector& model_by return m; } -svm_model *svm_load_model_from_bytes(const unsigned char *model_bytes, int size) { +svm_model *load_model_from_bytes(const unsigned char *model_bytes, int size) { const char* tmp_file_name = "tmp_svm_cpp.model"; std::ofstream out(tmp_file_name, std::ios::binary); out.write(reinterpret_cast(model_bytes), @@ -2858,10 +2879,8 @@ svm_model *svm_load_model_from_bytes(const unsigned char *model_bytes, int size) return m; } -svm_model *svm_load_model(const char *model_file_name) -{ - FILE *fp = fopen(model_file_name,"rb"); - if(fp==NULL) return NULL; + +svm_model* get_model_from_FILE_pointer(FILE *fp) { // read parameters @@ -3058,6 +3077,28 @@ svm_model *svm_load_model(const char *model_file_name) return model; } +svm_model *svm_load_model(const char *model_file_name) { + FILE *fp = fopen(model_file_name,"rb"); + if(fp == NULL) return NULL; + + return get_model_from_FILE_pointer(fp); +} + +svm_model *load_model_from_bytes(unsigned char* buffer, size_t size) { + FILE *fp = fmemopen(buffer, size, "r"); + if(fp == NULL) return NULL; + svm_model *model = get_model_from_FILE_pointer(fp); + + return model; +} + +svm_model *load_model_from_bytes_vector(std::vector * v){ + unsigned char* buf = v->data(); + svm_model * model = load_model_from_bytes(buf, v->size()); + return model; +} + + void svm_free_model_content(svm_model* model_ptr) { if(model_ptr->free_sv && model_ptr->l > 0 && model_ptr->SV != NULL) diff --git a/TPL/libsvm-3.1-custom/svm.def b/TPL/libsvm-3.1-custom/svm.def index 6d0cc7896..9e5db5696 100644 --- a/TPL/libsvm-3.1-custom/svm.def +++ b/TPL/libsvm-3.1-custom/svm.def @@ -18,3 +18,4 @@ EXPORTS svm_check_probability_model @16 svm_set_print_string_function @17 svm_load_model_from_bytes @18 + svm_conv_model_to_bytes @19 diff --git a/TPL/libsvm-3.1-custom/svm.h b/TPL/libsvm-3.1-custom/svm.h index 11e94305e..d24701328 100644 --- a/TPL/libsvm-3.1-custom/svm.h +++ b/TPL/libsvm-3.1-custom/svm.h @@ -99,6 +99,10 @@ const char *svm_check_parameter(const struct svm_problem *prob, const struct svm int svm_check_probability_model(const struct svm_model *model); void svm_set_print_string_function(void (*print_func)(const char *)); +void convert_model_to_bytes(svm_model *model, unsigned char* &buffer, size_t &size); +std::vector* convert_model_to_bytes_vector(svm_model *model); +struct svm_model *load_model_from_bytes(unsigned char* buffer, size_t size); +struct svm_model *load_model_from_bytes_vector(std::vector *v); #ifdef __cplusplus }