1 Matlab示例代码为两部分，分别对应不同的论文：
  2 
  3 1. Reducing the Dimensionality of data with neural networks 
  4 
  5 　　ministdeepauto.m   backprop.m   rbmhidlinear.m
  6 
  7 2. A fast learing algorithm for deep belief net
  8 
  9 　　mnistclassify.m 　　backpropclassfy.m  
 10 
 11  其余部分代码通用。
 12 
 13 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 14 mnistclassify.m
 15 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 16 
 17 clear all
 18 close all
 19 
 20 maxepoch=50; %迭代次数
 21 numhid=500; numpen=500; numpen2=2000;
 22 
 23 fprintf(1,‘Converting Raw files into Matlab format \n‘);
 24 converter;
 25 
 26 fprintf(1,‘Pretraining a deep autoencoder. \n‘);
 27 fprintf(1,‘The Science paper used 50 epochs. This uses %3i \n‘, maxepoch);
 28 
 29 makebatches;%分批数据 
 30 [numcases numdims numbatches]=size(batchdata); %获取batchdata数据大小
 31 %%numcases 每批数据的个数
 32 %%numdims 数据元组的维度
 33 %%numbtches 数据批数
 34 
 35 fprintf(1,‘Pretraining Layer 1 with RBM: %d-%d \n‘,numdims,numhid);%图像输入层到第一个隐藏层
 36 restart=1; %设置初始化参数
 37 rbm; %调用RBM训练数据 
 38 hidrecbiases=hidbiases; %获取隐藏层偏置值
 39 save mnistvhclassify vishid hidrecbiases visbiases; %
 40 
 41 fprintf(1,‘\nPretraining Layer 2 with RBM: %d-%d \n‘,numhid,numpen);%第一个隐藏层到第二个隐藏层
 42 batchdata=batchposhidprobs; %上一个RBM的隐藏层输出，读入作为这个RBM的输入
 43 numhid=numpen;%设置隐藏层的节点数，输入的节点数已经由读入数据给出
 44 restart=1;
 45 rbm;
 46 hidpen=vishid; penrecbiases=hidbiases; hidgenbiases=visbiases; %同上，提取权值，偏置，
 47 save mnisthpclassify hidpen penrecbiases hidgenbiases;
 48 
 49 fprintf(1,‘\nPretraining Layer 3 with RBM: %d-%d \n‘,numpen,numpen2);%第二个隐藏层到第三层隐藏层，其余同上
 50 batchdata=batchposhidprobs;
 51 numhid=numpen2;
 52 restart=1;
 53 rbm;
 54 hidpen2=vishid; penrecbiases2=hidbiases; hidgenbiases2=visbiases;
 55 save mnisthp2classify hidpen2 penrecbiases2 hidgenbiases2;
 56 
 57 backpropclassify;
 58 
 59  
 60 
 61 
 62 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 63 backpropclassify.m
 64 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 65 maxepoch=200;
 66 fprintf(1,‘\nTraining discriminative model on MNIST by minimizing cross entropy error. \n‘);%最小化交叉熵
 67 fprintf(1,‘60 batches of 1000 cases each. \n‘);
 68 
 69 load mnistvhclassify%加载各层之间的权值，以及偏置
 70 load mnisthpclassify
 71 load mnisthp2classify
 72 
 73 makebatches;%分批数据
 74 [numcases numdims numbatches]=size(batchdata);
 75 N=numcases; %获取每批数据向量数
 76 
 77 %%%% PREINITIALIZE WEIGHTS OF THE DISCRIMINATIVE MODEL%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 78 
 79 w1=[vishid; hidrecbiases];%第一层到第二层的权重，以及第二层的偏置
 80 w2=[hidpen; penrecbiases];%类上
 81 w3=[hidpen2; penrecbiases2];%类上
 82 w_class = 0.1*randn(size(w3,2)+1,10);%随机生成第四层列数+1行，10列的矩阵
 83 %%%%%%%%%% END OF PREINITIALIZATIO OF WEIGHTS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 84 
 85 l1=size(w1,1)-1;%获取每层的单元个数
 86 l2=size(w2,1)-1;
 87 l3=size(w3,1)-1;
 88 l4=size(w_class,1)-1;%最高层的单元个数
 89 l5=10; %label层单元个数
 90 test_err=[];%
 91 train_err=[];%
 92 
 93 
 94 for epoch = 1:maxepoch
 95 
 96 %%%%%%%%%%%%%%%%%%%% COMPUTE TRAINING MISCLASSIFICATION ERROR %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 97 err=0; 
 98 err_cr=0;
 99 counter=0;
100 [numcases numdims numbatches]=size(batchdata);
101 %%numcases 每批数据的个数
102 %%numdims 数据元组的维度
103 %%numbtches 数据批数
104 N=numcases;%%每批次数据向量个数
105 for batch = 1:numbatches
106 data = [batchdata(:,:,batch)];%读取一批次数据
107 target = [batchtargets(:,:,batch)];%读取当前批次的目标值
108 data = [data ones(N,1)];%在原数据后添加N行1列数据
109 w1probs = 1./(1 + exp(-data*w1)); w1probs = [w1probs ones(N,1)];%sigmod计算各层的概率值，参见BP算法
110 w2probs = 1./(1 + exp(-w1probs*w2)); w2probs = [w2probs ones(N,1)];
111 w3probs = 1./(1 + exp(-w2probs*w3)); w3probs = [w3probs ones(N,1)];
112 
113 targetout = exp(w3probs*w_class);%计算最后的输出值N行10列
114 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
115 %对最后的label的输出处理过程，见公式6.1，其中w3probs*w_class是label的输入
116 %最后只能有一个单元被激活，激活单元的选择即通过下面计算得出的概率来进行选择
117 %10个单元组成的“softmax”组
118 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
119 targetout = targetout./repmat(sum(targetout,2),1,10);%计算最后10个label输出除以输出值的总和
120 
121 [I J]=max(targetout,[],2);%取计算结果每行中的最大值,以及其列标
122 [I1 J1]=max(target,[],2);%取原先设定目标值的最大值以及列标
123 counter=counter+length(find(J==J1));%统计正确的条数
124 err_cr = err_cr- sum(sum( target(:,1:end).*log(targetout))) ; %%%%????
125 end
126 train_err(epoch)=(numcases*numbatches-counter);%总的错误条数？？？
127 train_crerr(epoch)=err_cr/numbatches;%平均每批次错误率？？？
128 
129 %%%%%%%%%%%%%% END OF COMPUTING TRAINING MISCLASSIFICATION ERROR %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
130 
131 %%%%%%%%%%%%%%%%%%%% COMPUTE TEST MISCLASSIFICATION ERROR %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
132 err=0;
133 err_cr=0;
134 counter=0;
135 [testnumcases testnumdims testnumbatches]=size(testbatchdata);
136 
137 N=testnumcases;
138 for batch = 1:testnumbatches
139 data = [testbatchdata(:,:,batch)];
140 target = [testbatchtargets(:,:,batch)];
141 data = [data ones(N,1)];
142 w1probs = 1./(1 + exp(-data*w1)); w1probs = [w1probs ones(N,1)];
143 w2probs = 1./(1 + exp(-w1probs*w2)); w2probs = [w2probs ones(N,1)];
144 w3probs = 1./(1 + exp(-w2probs*w3)); w3probs = [w3probs ones(N,1)];
145 targetout = exp(w3probs*w_class);
146 targetout = targetout./repmat(sum(targetout,2),1,10);
147 
148 [I J]=max(targetout,[],2);
149 [I1 J1]=max(target,[],2);
150 counter=counter+length(find(J==J1));
151 err_cr = err_cr- sum(sum( target(:,1:end).*log(targetout))) ;
152 end
153 test_err(epoch)=(testnumcases*testnumbatches-counter);
154 test_crerr(epoch)=err_cr/testnumbatches;
155 fprintf(1,‘Before epoch %d Train # misclassified: %d (from %d). Test # misclassified: %d (from %d) \t \t \n‘,...
156 epoch,train_err(epoch),numcases*numbatches,test_err(epoch),testnumcases*testnumbatches);
157 
158 %%%%%%%%%%%%%% END OF COMPUTING TEST MISCLASSIFICATION ERROR %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
159 
160 tt=0; 
161 for batch = 1:numbatches/10
162 fprintf(1,‘epoch %d batch %d\r‘,epoch,batch);
163 
164 %%%%%%%%%%% COMBINE 10 MINIBATCHES INTO 1 LARGER MINIBATCH %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
165 %组合10个小批次为1000样例的批次，然后用conjugate gradient来进行微调
166 tt=tt+1; 
167 data=[];
168 targets=[]; 
169 for kk=1:10
170 data=[data 
171 batchdata(:,:,(tt-1)*10+kk)]; %10个小批次合成
172 targets=[targets
173 batchtargets(:,:,(tt-1)*10+kk)];
174 end
175 
176 %%%%%%%%%%%%%%% PERFORM CONJUGATE GRADIENT WITH 3 LINESEARCHES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
177 max_iter=3; %设置线性搜索的次数
178 
179 if epoch<6 % First update top-level weights holding other weights fixed. 
180 N = size(data,1); %获取数据的行数 
181 XX = [data ones(N,1)]; %每行数据后面增加1，用来增加偏置
182 w1probs = 1./(1 + exp(-XX*w1)); w1probs = [w1probs ones(N,1)];
183 w2probs = 1./(1 + exp(-w1probs*w2)); w2probs = [w2probs ones(N,1)];
184 w3probs = 1./(1 + exp(-w2probs*w3)); %w3probs = [w3probs ones(N,1)];
185 
186 VV = [w_class(:)‘]‘; %VV将随机生成的向量w_class展开成一列？？？为什么展开成一列与minimize的参数有关
187 %
188 Dim = [l4; l5]; %记录最后两层的单元节点数，即2000的隐藏层和10的label层
189 [X, fX] = minimize(VV,‘CG_CLASSIFY_INIT‘,max_iter,Dim,w3probs,targets);%只训练两层 %%%详细见函数定义
190 %minimize is Cari Rasmusssen‘s "minimize" code
191 %%------------------参数含义------------------%%
192 %VV 随机权重向量的展开 ，其作为输入参数，列必须为1（D by 1) 
193 %X 函数f="CG_CLASSIFY_INIT"的最优化参数
194 %fX 函数f对X的偏导
195 %max_iter 如果为正，表示线性搜索次数，为负，函数的最大值个数
196 %%-------------------------------------------------%
197 w_class = reshape(X,l4+1,l5);%恢复权值矩阵结构
198 
199 else %进入整体微调过程
200 VV = [w1(:)‘ w2(:)‘ w3(:)‘ w_class(:)‘]‘; %将所有权值按列展开成一列
201 Dim = [l1; l2; l3; l4; l5]; %记录各层单元个数传入
202 [X, fX] = minimize(VV,‘CG_CLASSIFY‘,max_iter,Dim,data,targets);
203 
204 w1 = reshape(X(1:(l1+1)*l2),l1+1,l2); %恢复W1权值1.0
205 xxx = (l1+1)*l2; %临时变量，用于恢复权值单元
206 w2 = reshape(X(xxx+1:xxx+(l2+1)*l3),l2+1,l3);
207 xxx = xxx+(l2+1)*l3;
208 w3 = reshape(X(xxx+1:xxx+(l3+1)*l4),l3+1,l4);
209 xxx = xxx+(l3+1)*l4;
210 w_class = reshape(X(xxx+1:xxx+(l4+1)*l5),l4+1,l5);
211 
212 end
213 %%%%%%%%%%%%%%% END OF CONJUGATE GRADIENT WITH 3 LINESEARCHES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
214 
215 end
216 
217 save mnistclassify_weights w1 w2 w3 w_class
218 save mnistclassify_error test_err test_crerr train_err train_crerr;
219 
220 end
221 
222  
223 
224 
225 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
226 rbm.m
227 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%228 epsilonw = 0.1; % Learning rate for weights 
229 epsilonvb = 0.1; % Learning rate for biases of visible units 
230 epsilonhb = 0.1; % Learning rate for biases of hidden units 
231 weightcost = 0.0002; 
232 initialmomentum = 0.5;
233 finalmomentum = 0.9;
234 
235 [numcases numdims numbatches]=size(batchdata);
236 %%numcases 每批数据的个数
237 %%numdims 数据元组的维度
238 %%numbtches 数据批数
239 
240 if restart ==1,
241 restart=0;
242 epoch=1;
243 
244 % Initializing symmetric weights and biases. 初始化对称权值和偏置
245 vishid = 0.1*randn(numdims, numhid); %初始化生成可视层到隐藏层的权值
246 hidbiases = zeros(1,numhid);%隐藏单元的偏置值
247 visbiases = zeros(1,numdims);%可见单元的偏置值
248 
249 poshidprobs = zeros(numcases,numhid); %正向的隐藏单元概率生成
250 neghidprobs = zeros(numcases,numhid);%反向的隐藏单元概率生成
251 posprods = zeros(numdims,numhid);%正向可见单元概率生成
252 negprods = zeros(numdims,numhid);%反向可见单元概率生成
253 vishidinc = zeros(numdims,numhid);%%%%%可视单元和隐藏单元之间的权值增量
254 hidbiasinc = zeros(1,numhid);%%隐藏单元的偏置增量
255 visbiasinc = zeros(1,numdims);%%可视单元的偏置增量
256 batchposhidprobs=zeros(numcases,numhid,numbatches);%存储每次迭代计算好的每层的隐藏层概率，作为下一个RBM的输入
257 end
258 
259 %%%%%%%%%%%%%%%%简单输出 迭代次数 处理的批次%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
260 for epoch = epoch:maxepoch, %迭代处理
261 fprintf(1,‘epoch %d\r‘,epoch); 
262 errsum=0; %初始化输出错误为0
263 for batch = 1:numbatches, %每次处理一批次的数据
264 fprintf(1,‘epoch %d batch %d\r‘,epoch,batch);
265 
266 %%%%%%%%% START POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
267 data = batchdata(:,:,batch); %读取当前批次的全部数据vi
268 poshidprobs = 1./(1 + exp(-data*vishid - repmat(hidbiases,numcases,1))); %计算前向传播的隐藏层概率hi
269 batchposhidprobs(:,:,batch)=poshidprobs;%将计算好的概率赋值给当前批次前向传播的隐藏层最后一次计算好的值作为下一层的输入
270 posprods = data‘ * poshidprobs;%contrastive divergence过程<vi,hi>
271 
272 poshidact = sum(poshidprobs);%average-wise隐藏层激活概率值
273 posvisact = sum(data);%average-wise可视层激活概率值
274 
275 %%%%%%%%% END OF POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
276 poshidstates = poshidprobs > rand(numcases,numhid);%gibbs抽样,设定状态
277 
278 %%%%%%%%% START NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
279 negdata = 1./(1 + exp(-poshidstates*vishid‘ - repmat(visbiases,numcases,1)));%根据hi计算vi+1
280 neghidprobs = 1./(1 + exp(-negdata*vishid - repmat(hidbiases,numcases,1))); %根据vi+1计算hi+1
281 negprods = negdata‘*neghidprobs;%contrastive divergence <vi+1,hi+1>
282 
283 neghidact = sum(neghidprobs);
284 negvisact = sum(negdata);
285 
286 %%%%%%%%% END OF NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
287 err= sum(sum( (data-negdata).^2 )); %重新构建数据的方差
288 errsum = err + errsum;%整体方差
289 
290 if epoch>5, %迭代次数不同调整冲量
291 momentum=finalmomentum;
292 else
293 momentum=initialmomentum;
294 end;
295 
296 %%%%%%%%% UPDATE WEIGHTS AND BIASES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 
297 vishidinc = momentum*vishidinc + ...
298 epsilonw*( (posprods-negprods)/numcases - weightcost*vishid);%权重增量计算
299 visbiasinc = momentum*visbiasinc + (epsilonvb/numcases)*(posvisact-negvisact);%偏置增量计算
300 hidbiasinc = momentum*hidbiasinc + (epsilonhb/numcases)*(poshidact-neghidact);%隐藏层增量计算
301 
302 vishid = vishid + vishidinc;
303 visbiases = visbiases + visbiasinc;
304 hidbiases = hidbiases + hidbiasinc;
305 
306 %%%%%%%%%%%%%%%% END OF UPDATES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
307 
308 end
309 fprintf(1, ‘epoch %4i error %6.1f \n‘, epoch, errsum); 
310 end;
311 
312  
313 
314 
315 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
316 CG_CLASSIFY_INIT.M
317 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%318 function [f, df] = CG_CLASSIFY_INIT(VV,Dim,w3probs,target);%CG对最上面两层的训练
319 l1 = Dim(1);
320 l2 = Dim(2);
321 N = size(w3probs,1); 
322 % Do decomversion.
323 w_class = reshape(VV,l1+1,l2); %恢复权重，
324 w3probs = [w3probs ones(N,1)]; %一列，偏置
325 
326 targetout = exp(w3probs*w_class); %计算label层的输出结果为numbercase＊lablesnumber的矩阵
327 targetout = targetout./repmat(sum(targetout,2),1,10); %选择最后的激活单元，见backpropclassify.m 的76行
328 f = -sum(sum( target(:,1:end).*log(targetout))) ; %交叉熵 只采用了前边部分
329 
330 IO = (targetout-target(:,1:end)); % 输入和输出结果之间的差值
331 Ix_class=IO; %
332 dw_class = w3probs‘*Ix_class;%导数F(x)((1-F(x))乘以输出结果的偏差..其中F为sigmoid函数
333 
334 df = [dw_class(:)‘]‘;
335 
336  
337 
338  
339 
340 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
341 CG_CLASSIFY.M
342 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
343 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
344 % 该段代码对所有权重进行整体微调
345 % 各部分过程见 CG_CLASSIFY_INIT.m注解
346 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
347 function [f, df] = CG_CLASSIFY(VV,Dim,XX,target);
348 
349 
350 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
351 rbmhidlinear.m
352 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
353 %除了最后计算单元值采用的是线性单元其余过程全部一样
354 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
355 
356 复制代码