-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathbuild_notes_data.txt
219 lines (190 loc) · 8.22 KB
/
build_notes_data.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
Install freezing at Partition
Switch SATA Configuration from RAID on to AHCI
CUDA
become a member of the Nvidia developer community
wget URL
start computer in low graphics mode
sudo apt-get purge nvidia*
sudo add-apt-repository ppa:graphics-drivers/ppa
sudo apt-get update
sudo ubuntu-drivers devices
sudo apt-get install nvidia-381
sudo ldconfig
CUDNN
wget URL
sudo cp -i include/cudnn.h /usr/local/cuda-8.0/include/
sudo cp -i lib64/libcudnn* /usr/local/cuda-8.0/lib64/
TensorFlow
wget https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.12.1-cp27-none-linux_x86_64.whl
sudo apt-get install python-pip python-dev
sudo pip install tensorflow
[or]
pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp36-cp36m-linux_x86_64.whl
Test
python
>>> import tensorflow as tf
>>> hello = tf.constant('Hello, TensorFlow!')
>>> sess = tf.Session()
>>> print(sess.run(hello))
Hello, TensorFlow!
sudo pip install numpy scipy scikit-learn pillow matplotlib h5py keras
sudo apt-get install -y python-tk ipython
sudo apt-get install -y libopencv-dev python-opencv
sudo apt-get install -y build-essential libxmu-dev libxmu6 libxi-dev libxine2-dev libalut-dev freeglut3 freeglut3-dev cmake libogg-dev libvorbis-dev libxxf86dga-dev libxxf86vm-dev libxrender-dev libxrandr-dev zlib1g-dev libpng12-dev
sudo apt-get install -y libglib2.0-dev libgl1-mesa-dev libglu1-mesa-dev freeglut3-dev libplib-dev libopenal-dev libalut-dev libxi-dev libxmu-dev libxrender-dev libxrandr-dev libpng12-dev
wget http://plib.sourceforge.net/dist/plib-1.8.5.tar.gz
tar xzf plib-1.8.5.tar.gz
cd plib-1.8.5
./configure CFLAGS="-O2 -m64 -fPIC" CPPFLAGS="-O2 -fPIC" CXXFLAGS="-O2 -fPIC" LDFLAGS="-L/usr/lib64"
sudo make install
wget https://github.com/kcat/openal-soft/archive/openal-soft-1.13.tar.gz
tar zxf openal-soft-1.13.tar.gz
cd openal-soft-openal-soft-1.13
cd cmake
cmake ..
make
sudo make install
python3 -V
sudo apt-get install -y python3-pip
sudo apt-get install -y libssl-dev libffi-dev python-dev
sudo pip3 install numpy scipy scikit-learn
gym_torcs
sudo apt-get install -y python-numpy python-dev cmake zlib1g-dev libjpeg-dev xvfb libav-tools xorg-dev python-opengl libboost-all-dev libsdl2-dev swig xautomation xvfb git libglib2.0-dev libgl1-mesa-dev libglu1-mesa-dev freeglut3-dev libplib-dev libopenal-dev libalut-dev libxi-dev libxmu-dev libxrender-dev libxrandr-dev libpng12-dev
sudo apt-get install zlib1g-dev libjpeg-dev xvfb libav-tools xorg-dev python-opengl libsdl2-dev swig xautomation xvfb libglib2.0-dev libgl1-mesa-dev libglu1-mesa-dev freeglut3-dev libplib-dev libopenal-dev libalut-dev libxi-dev libxmu-dev libxrender-dev libxrandr-dev libpng12-dev
sudo pip install 'gym[all]'
git clone https://github.com/ugo-nama-kun/gym_torcs.git
cd gym_torcs/vtorcs-RL-color/
./configure
make
sudo make install
sudo make datainstall
# run at least once
sudo torcs
[or] DeepDriving Torcs
cd DeepDrivingCode/torcs-1.3.6
./configure
make
sudo make install
sudo make datainstall
# test
torcs
cd /usr/local/share/games
sudo chown -R asankar:asankar torcs
cd /usr/local/lib
sudo chown -R asankar:asankar torcs
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
export TORCS_BASE=/home/asankar/deepdrive/DeepDrivingCode/torcs-1.3.6
export MAKE_DEFAULT=$TORCS_BASE/Make-default.mk
cp -r modified_tracks/* /usr/local/share/games/torcs/tracks/road/
[install caffe prereqs]
cd Caffe_driving
make
cp -r modified_tracks/* /usr/local/share/games/torcs/tracks/road/
Delete Torcs
sudo rm -rf /usr/local/lib/torcs /usr/local/bin/torcs /usr/local/share/games/torcs /home/asankar/.torcs
Boost
conda install boost
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/asankar/anaconda3/lib
conda install nomkl numpy scipy scikit-learn numexpr
conda remove mkl mkl-service
Protobuf
wget https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip
wget https://github.com/google/protobuf/releases/download/v3.2.0/protobuf-python-3.2.0.zip
wget https://github.com/google/protobuf/releases/download/v3.2.0/protobuf-cpp-3.2.0.zip
sudo apt-get install autoconf automake libtool curl make g++ unzip
unzip protoc-3.2.0-linux-x86_64.zip -d protoc
sudo cp bin/protoc /usr/bin/
unzip protobuf-cpp-3.2.0.zip
cd protobuf-cpp/protobuf-3.2.0
./configure
make
make check
sudo make install
sudo ldconfig
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
unzip protobuf-python-3.2.0.zip -d protobuf-python
cd protobuf-python/protobuf-3.2.0
python setup.py build
python setup.py test
OpenCV 2
sudo apt-get install -y libjpeg8-dev libtiff4-dev libjasper-dev libpng12-dev libavcodec-dev libavformat-dev libswscale-dev libv4l-dev libgtk2.0-dev libatlas-base-dev gfortran
pip install numpy
sudo apt-get install libopencv-dev python-opencv
Caffe
sudo apt-get install -y libleveldb-dev libsnappy-dev libopencv-dev libhdf5-serial-dev
sudo apt-get install --no-install-recommends libboost-all-dev
sudo apt-get install -y libatlas-base-dev libgflags-dev libgoogle-glog-dev liblmdb-dev
(old protobuf only)
sudo apt-get install -y libprotobuf-dev protobuf-compiler
git clone https://github.com/BVLC/caffe.git
cd caffe
make all
make install
make runtest
cd python
for req in $(cat requirements.txt); do pip install $req; done
cd ..
make pycaffe
export PYTHONPATH=$PYTHONPATH:/home/asankar/deepdrive/caffe36/python
sudo apt-get install libtiff4-dev
exclude anaconda3/lib from LD_LIBRARY_PATH during make all for libopencv problem
edit requirements so that dateutil>=2.0
LevelDB
sudo apt-get install libleveldb1 libleveldb-dev
sudo pip install plyvel
pip install --user cython h5py
pip install Pillow
pip install keras
git clone https://github.com/heuritech/convnets-keras.git
cd convnets-keras
sudo /home/asankar/anaconda3/bin/python3.6 setup.py install
Convert Caffe Weights
git clone https://github.com/ethereon/caffe-tensorflow.git
wget http://deepdriving.cs.princeton.edu/DeepDrivingCode_v2.zip
unzip DeepDrivingCode_v2.zip
mkdir models
cp DeepDrivingCode/Caffe_driving/torcs/pre_trained/driving_* models/
cd caffe36/build/tools
./upgrade_net_proto_binary ../../../models/driving_train_1F_iter_140000.caffemodel ../../../models/driving_train_1F_iter_140000_new.binaryproto
./upgrade_net_proto_text ../../../models/driving_run_1F.prototxt ../../../models/driving_run_1F_new.prototxt
cd ../../../caffe-tensorflow
vi kaffe/graph.py +145
# change rb to r
vi kaffe/graph.py +124
# add !s to last 2 string format sections
vi kaffe/graph.py +122
vi kaffe/transformers.py +127
vi kaffe/transformers.py +291
# convert to list list(node.data)
cp ../models/driving_run_1F_new.prototxt ../models/driving_run_1F_new2.prototxt
# change first data layer to an input layer (3x280x210)
python convert.py --caffemodel=../models/driving_train_1F_iter_140000_new.binaryproto --code-output-path=../models/caffe_alexnet.py --data-output-path=../models/alexnet.h5 ../models/driving_run_1F_new2.prototxt
Convert Caffe Weights 2
sudo add-apt-repository ppa:george-edison55/cmake-3.x
sudo apt-get update
sudo apt-get install cmake
sudo pip install Theano
cd libgpuarray
mkdir build && cd build
cmake .. -DCMAKE_BUILD_TYPE=Release
make
sudo make install
cd ..
python setup.py build
python setup.py install
git clone https://github.com/MarcBS/keras.git
export PYTHONPATH=$PYTHONPATH:/home/asankar/deepdrive/keras
python2.7 caffe2keras.py -load_path '../models/' -prototxt 'driving_train_1F.prototxt' -caffemodel 'driving_train_1F_iter_140000.caffemodel'
In total, we collect 484,815 images for training. The
training procedure is similar to training an AlexNet on ImageNet
data. The differences are: the input image has a resolution
of 280 × 210 and is no longer a square image. We do
not use any crops or a mirrored version. We train our model
from scratch. We choose an initial learning rate of 0.01, and
each mini-batch consists of 64 images randomly selected
from the training samples. After 140,000 iterations, we stop
the training process.
Example: if you have 1000 training examples, and your batch size is 500, then it will take 2 iterations to complete 1 epoch.
19 epochs
export PYTHONPATH=/home/asankar/deepdrive/caffe36/python
export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/lib:/home/asankar/anaconda3/lib