fix bug in thread release

build/CMakeFiles/CMakeError.log

@@ -1,22 +1,22 @@
 Determining if the pthread_create exist failed with the following output:
 Change Dir: /home/cs/Desktop/cs_work/sita/build/CMakeFiles/CMakeTmp
 
-Run Build Command:"/usr/bin/make" "cmTC_1fea6/fast"
-/usr/bin/make -f CMakeFiles/cmTC_1fea6.dir/build.make CMakeFiles/cmTC_1fea6.dir/build
+Run Build Command:"/usr/bin/make" "cmTC_54c0c/fast"
+/usr/bin/make -f CMakeFiles/cmTC_54c0c.dir/build.make CMakeFiles/cmTC_54c0c.dir/build
 make[1]: Entering directory '/home/cs/Desktop/cs_work/sita/build/CMakeFiles/CMakeTmp'
-Building C object CMakeFiles/cmTC_1fea6.dir/CheckSymbolExists.c.o
-/usr/bin/cc -fPIC -o CMakeFiles/cmTC_1fea6.dir/CheckSymbolExists.c.o -c /home/cs/Desktop/cs_work/sita/build/CMakeFiles/CMakeTmp/CheckSymbolExists.c
-Linking C executable cmTC_1fea6
-/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_1fea6.dir/link.txt --verbose=1
-/usr/bin/cc -fPIC CMakeFiles/cmTC_1fea6.dir/CheckSymbolExists.c.o -o cmTC_1fea6 
-CMakeFiles/cmTC_1fea6.dir/CheckSymbolExists.c.o: In function `main':
+Building C object CMakeFiles/cmTC_54c0c.dir/CheckSymbolExists.c.o
+/usr/bin/cc -fPIC -o CMakeFiles/cmTC_54c0c.dir/CheckSymbolExists.c.o -c /home/cs/Desktop/cs_work/sita/build/CMakeFiles/CMakeTmp/CheckSymbolExists.c
+Linking C executable cmTC_54c0c
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_54c0c.dir/link.txt --verbose=1
+/usr/bin/cc -fPIC CMakeFiles/cmTC_54c0c.dir/CheckSymbolExists.c.o -o cmTC_54c0c 
+CMakeFiles/cmTC_54c0c.dir/CheckSymbolExists.c.o: In function `main':
 CheckSymbolExists.c:(.text+0x1b): undefined reference to `pthread_create'
 collect2: error: ld returned 1 exit status
-CMakeFiles/cmTC_1fea6.dir/build.make:97: recipe for target 'cmTC_1fea6' failed
-make[1]: *** [cmTC_1fea6] Error 1
+CMakeFiles/cmTC_54c0c.dir/build.make:97: recipe for target 'cmTC_54c0c' failed
+make[1]: *** [cmTC_54c0c] Error 1
 make[1]: Leaving directory '/home/cs/Desktop/cs_work/sita/build/CMakeFiles/CMakeTmp'
-Makefile:126: recipe for target 'cmTC_1fea6/fast' failed
-make: *** [cmTC_1fea6/fast] Error 2
+Makefile:126: recipe for target 'cmTC_54c0c/fast' failed
+make: *** [cmTC_54c0c/fast] Error 2
 
 File /home/cs/Desktop/cs_work/sita/build/CMakeFiles/CMakeTmp/CheckSymbolExists.c:
 /* */
@@ -36,20 +36,20 @@ int main(int argc, char** argv)
 Determining if the function pthread_create exists in the pthreads failed with the following output:
 Change Dir: /home/cs/Desktop/cs_work/sita/build/CMakeFiles/CMakeTmp
 
-Run Build Command:"/usr/bin/make" "cmTC_65616/fast"
-/usr/bin/make -f CMakeFiles/cmTC_65616.dir/build.make CMakeFiles/cmTC_65616.dir/build
+Run Build Command:"/usr/bin/make" "cmTC_20520/fast"
+/usr/bin/make -f CMakeFiles/cmTC_20520.dir/build.make CMakeFiles/cmTC_20520.dir/build
 make[1]: Entering directory '/home/cs/Desktop/cs_work/sita/build/CMakeFiles/CMakeTmp'
-Building C object CMakeFiles/cmTC_65616.dir/CheckFunctionExists.c.o
-/usr/bin/cc -fPIC -DCHECK_FUNCTION_EXISTS=pthread_create -o CMakeFiles/cmTC_65616.dir/CheckFunctionExists.c.o -c /usr/share/cmake-3.5/Modules/CheckFunctionExists.c
-Linking C executable cmTC_65616
-/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_65616.dir/link.txt --verbose=1
-/usr/bin/cc -fPIC -DCHECK_FUNCTION_EXISTS=pthread_create CMakeFiles/cmTC_65616.dir/CheckFunctionExists.c.o -o cmTC_65616 -lpthreads 
+Building C object CMakeFiles/cmTC_20520.dir/CheckFunctionExists.c.o
+/usr/bin/cc -fPIC -DCHECK_FUNCTION_EXISTS=pthread_create -o CMakeFiles/cmTC_20520.dir/CheckFunctionExists.c.o -c /usr/share/cmake-3.5/Modules/CheckFunctionExists.c
+Linking C executable cmTC_20520
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_20520.dir/link.txt --verbose=1
+/usr/bin/cc -fPIC -DCHECK_FUNCTION_EXISTS=pthread_create CMakeFiles/cmTC_20520.dir/CheckFunctionExists.c.o -o cmTC_20520 -lpthreads 
 /usr/bin/ld: cannot find -lpthreads
 collect2: error: ld returned 1 exit status
-CMakeFiles/cmTC_65616.dir/build.make:97: recipe for target 'cmTC_65616' failed
-make[1]: *** [cmTC_65616] Error 1
+CMakeFiles/cmTC_20520.dir/build.make:97: recipe for target 'cmTC_20520' failed
+make[1]: *** [cmTC_20520] Error 1
 make[1]: Leaving directory '/home/cs/Desktop/cs_work/sita/build/CMakeFiles/CMakeTmp'
-Makefile:126: recipe for target 'cmTC_65616/fast' failed
-make: *** [cmTC_65616/fast] Error 2
+Makefile:126: recipe for target 'cmTC_20520/fast' failed
+make: *** [cmTC_20520/fast] Error 2
 
 

include/sita/dataprovider/dataprovider.h

@@ -36,7 +36,7 @@ class DataProviderEntry: public InternalThread{
 template <typename Dtype>
 class DataProvider{
 public:
- DataProvider(std::string data_file, std::string label_file, std::vector<Dtype> means, int batch_size, int thread_num){
+ DataProvider(std::string data_file, std::string label_file, std::vector<Dtype> means, int batch_size, int thread_num, bool shuffle){
  };
  ~DataProvider(){};
  static const int PREFETCH_COUNT = 3;

include/sita/dataprovider/mnist_dataprovider.h

@@ -52,8 +52,13 @@ template <typename Dtype>
 class MnistDataProvider: public DataProvider<Dtype>{
 public:
  MnistDataProvider(std::string data_file, std::string label_file,
- std::vector<Dtype> means, int batch_size, int thread_num);
- ~MnistDataProvider(){}
+ std::vector<Dtype> means, int batch_size, int thread_num,bool shuffle);
+ ~MnistDataProvider(){
+ for(int i = 0; i < _threads.size(); i++){
+ _threads[i].stop_internal_thread();//join main processor
+ }
+
+ }
 
  MnistBatch<Dtype> * fetch_batch();
 

include/sita/stuff/rng/rng.h

@@ -0,0 +1,44 @@
+//
+// Created by unsky on 09/08/18.
+//
+
+#ifndef SITA_RNG_H
+#define SITA_RNG_H
+
+#include <algorithm>
+#include <iterator>
+
+#include "boost/random/mersenne_twister.hpp"
+#include "boost/random/uniform_int.hpp"
+
+namespace sita{
+
+typedef boost::mt19937 rng_t;
+
+inline rng_t* caffe_rng() {
+ return static_cast<rng_t*>(Caffe::rng_stream().generator());
+}
+
+// Fisher–Yates algorithm
+template <class RandomAccessIterator, class RandomGenerator>
+inline void shuffle(RandomAccessIterator begin, RandomAccessIterator end,
+ RandomGenerator* gen) {
+ typedef typename std::iterator_traits<RandomAccessIterator>::difference_type
+ difference_type;
+ typedef typename boost::uniform_int<difference_type> dist_type;
+
+ difference_type length = std::distance(begin, end);
+ if (length <= 0) return;
+
+ for (difference_type i = length - 1; i > 0; --i) {
+ dist_type dist(0, i);
+ std::iter_swap(begin + i, begin + dist(*gen));
+ }
+}
+
+template <class RandomAccessIterator>
+inline void shuffle(RandomAccessIterator begin, RandomAccessIterator end) {
+ shuffle(begin, end, caffe_rng());
+}
+}
+#endif //SITA_RNG_H

include/sita/stuff/stuff.h

@@ -0,0 +1,23 @@
+//
+// Created by unsky on 09/08/18.
+//
+
+#ifndef SITA_STUFF_STUFF_H
+#define SITA_STUFF_STUFF_H
+namespace sita {
+
+class Stuff {
+public:
+
+ Stuff() {}
+ ~Stuff() {}
+
+ virtual void inferance() {}
+
+ virtual void backward() {}
+
+private:
+ DISABLE_COPY_AND_ASSIGN(Stuff);
+};
+}
+#endif //SITA_STUFF_H

main.cpp

@@ -23,11 +23,11 @@ int main(int argc, char** argv) {
  means.push_back(float(5));
  means.push_back(float(10));
  sita::MnistDataProvider<float > mnistdp("../data/mnist/train-images-idx3-ubyte",
- "../data/mnist/train-labels-idx1-ubyte",means,10,5);
+ "../data/mnist/train-labels-idx1-ubyte",means,1,1,true);
 
  int k = 0;
 
- while(k != 3000) {
+ while(k != 3) {
  k++;
  sita::MnistBatch<float> * batch = mnistdp.fetch_batch();
  // LOG(INFO)<<batch->label()->cpu_data()[0];
@@ -51,7 +51,7 @@ int main(int argc, char** argv) {
  }
  gws.train();
  }
- //workspace test
+
 
 
  return 0;

src/sita/dataprovider/mnist_dataprovider.cpp

@@ -6,8 +6,8 @@ namespace sita{
 
 template <typename Dtype>
 MnistDataProvider<Dtype>::MnistDataProvider(std::string data_file, std::string label_file,
- std::vector<Dtype> means, int batch_size, int thread_num):DataProvider<Dtype>(data_file,
- label_file, means, batch_size, thread_num) {
+ std::vector<Dtype> means, int batch_size, int thread_num, bool shuffle):DataProvider<Dtype>(data_file,
+ label_file, means, batch_size, thread_num, shuffle) {
  LOG(INFO) << "loading mnist dataset using "<< thread_num <<" threads......";
  _threads.resize(thread_num);
  _thread_images.resize(thread_num);
@@ -22,6 +22,13 @@ MnistDataProvider<Dtype>::MnistDataProvider(std::string data_file, std::string l
 
  CHECK_EQ(_images.size(), _labels.size()) << "label size do not equal image size in mnist!!";
 
+ if(shuffle){
+
+
+
+
+ }
+
  int num_images_in_one_thread = _images.size()/thread_num - 1;
 
  LOG(INFO) << num_images_in_one_thread << " data processing in each thread !!";

src/sita/internal_thread.cpp

@@ -32,6 +32,7 @@ namespace sita {
  }
 
  void InternalThread::stop_internal_thread() {
+ //fist thread try to join the main and interupted itself
  if (is_started()) {
  thread_->interrupt();
  try {
@@ -41,6 +42,10 @@ namespace sita {
  LOG(FATAL) << "Thread exception: " << e.what();
  }
  }
+ //second third,..thread interrupt
+ if(thread_){
+ thread_->interrupt();
+ }
  }
 
 } // namespace sita