Add files via upload

hybnet/hnetLSTMlayer.m

@@ -0,0 +1,171 @@
+function layer=hnetLSTMlayer(ninputs,numnodes,inpl)%#######################
+if nargin<3
+    inpl=-1;
+end
+layer.type = 'LSTM';
+layer.nx = ninputs;
+layer.ny = numnodes;
+layer.inpl = inpl; 
+layer.y=zeros(numnodes,1);
+layer.cold = zeros(numnodes,1);
+layer.dcdf= zeros(numnodes,1);
+layer.dcdi= zeros(numnodes,1);
+layer.dcdcest= zeros(numnodes,1);
+layer.c=zeros(layer.ny,1);
+
+layer.fg = hnetfflayer(ninputs+numnodes,numnodes,'sigm');
+layer.ig = hnetfflayer(ninputs+numnodes,numnodes,'sigm');
+layer.og = hnetfflayer(ninputs+numnodes,numnodes,'sigm');
+layer.ci = hnetfflayer(ninputs,numnodes,'tanh');
+
+layer.nw = layer.fg.nw+layer.ig.nw+layer.og.nw+layer.ci.nw;
+layer.initwfun = @lstminitw;
+layer.setwfun = @lstmsetw;
+layer.getwfun = @lstmgetw;
+layer.getgradfun = @lstmgetgrad;
+layer.fffun = @lstmff;
+layer.bpfun = @lstmbp;
+layer.bpresetfun = @lstmresetbp;
+layer.updwfun=@lstmupdweights;
+layer.initstatefun=@lstminitstate;
+end
+
+function layer=lstminitw(layer,dw)%########################################
+if nargin<2
+    dw=2; %%%1 Normal  %%%2 Xavier
+end
+
+layer.fg = feval(layer.fg.initwfun,layer.fg,dw);
+layer.ig = feval(layer.ig.initwfun,layer.ig,dw);
+layer.og = feval(layer.og.initwfun,layer.og,dw);
+layer.ci = feval(layer.ci.initwfun,layer.ci,dw);
+
+end
+
+function layer=lstminitstate(layer)%########################################
+layer.c=zeros(layer.ny,1);
+layer.cold=zeros(layer.ny,1);
+layer.y=zeros(layer.ny,1);
+layer.dcdf= zeros(layer.ny,1);
+layer.dcdi= zeros(layer.ny,1);
+layer.dcdcest= zeros(layer.ny,1);
+end
+
+function layer=lstmsetw(layer,w)%############################################
+count = 1;
+layer.fg = feval(layer.fg.setwfun,layer.fg,w(count:count+layer.fg.nw-1,1));
+count = count + layer.fg.nw;
+layer.ig = feval(layer.ig.setwfun,layer.ig,w(count:count+layer.ig.nw-1,1));
+count = count + layer.ig.nw;
+layer.og = feval(layer.og.setwfun,layer.og,w(count:count+layer.og.nw-1,1));
+count = count + layer.og.nw;
+layer.ci = feval(layer.ci.setwfun,layer.ci,w(count:count+layer.ci.nw-1,1));
+end
+
+function w=lstmgetw(layer)%################################################
+w = zeros(layer.nw,1);
+count = 1;
+w(count:count+layer.fg.nw-1,1) = feval(layer.fg.getwfun,layer.fg);
+count = count + layer.fg.nw;
+w(count:count+layer.ig.nw-1,1) = feval(layer.ig.getwfun,layer.ig);
+count = count + layer.ig.nw;
+w(count:count+layer.og.nw-1,1) = feval(layer.og.getwfun,layer.og);
+count = count + layer.og.nw;
+w(count:count+layer.ci.nw-1,1) = feval(layer.ci.getwfun,layer.ci);
+end
+
+function [dlossdw,sum_dlossdw]=lstmgetgrad(layer)%#########################
+dlossdw=zeros(layer.nw,1);
+sum_dlossdw=zeros(layer.nw,1);
+count = 1;
+[dlossdw(count:count+layer.fg.nw-1,1),...
+    sum_dlossdw(count:count+layer.fg.nw-1,1)] = ...
+    feval(layer.fg.getgradfun,layer.fg);
+count = count + layer.fg.nw;
+[dlossdw(count:count+layer.ig.nw-1,1),...
+    sum_dlossdw(count:count+layer.ig.nw-1,1)] = ...
+    feval(layer.ig.getgradfun,layer.ig);
+count = count + layer.ig.nw;
+[dlossdw(count:count+layer.og.nw-1,1),...
+    sum_dlossdw(count:count+layer.og.nw-1,1)] = ...
+    feval(layer.og.getgradfun,layer.og);
+count = count + layer.og.nw;
+[dlossdw(count:count+layer.ci.nw-1,1),...
+    sum_dlossdw(count:count+layer.ci.nw-1,1)] = ...
+    feval(layer.ci.getgradfun,layer.ci);
+end
+
+function layer=lstmff(layer,x)%############################################
+layer.cold=layer.c;  %??????
+
+layer.x=x;
+
+inp = [x;layer.cold];
+
+layer.fg = feval(layer.fg.fffun,layer.fg,inp);
+layer.f = layer.fg.y;
+layer.ig = feval(layer.ig.fffun,layer.ig,inp);
+layer.i=layer.ig.y;
+layer.og = feval(layer.og.fffun,layer.og,inp);
+layer.o = layer.og.y; 
+layer.ci = feval(layer.ci.fffun,layer.ci,x);
+layer.cest = layer.ci.y;
+
+layer.c= layer.f.*layer.cold + layer.i.*layer.cest;
+
+layer.y = layer.o.*layer.c;
+
+layer.dcdf=      layer.f.*layer.dcdf    +   layer.cold;
+layer.dcdi=      layer.f.*layer.dcdi    +   layer.cest;
+layer.dcdcest=   layer.f.*layer.dcdcest +   layer.i;
+
+% layer.dcdf=      layer.cold;
+% layer.dcdi=      layer.cest;
+% layer.dcdcest=   layer.i;
+
+end
+
+function layer=lstmbp(layer,dlossdy)%########################################
+%y = o.*c; ------
+dlossdo = dlossdy.*layer.c;
+dlossdc = dlossdy.*layer.o;
+%c= f.*cold + i.*cest; ----------
+dlossdf = dlossdc.*layer.dcdf;
+dlossdi = dlossdc.*layer.dcdi;
+dlossdcest = dlossdc.*layer.dcdcest;
+
+%layer.ci = feval(layer.ci.fffun,layer.ci,inp);
+layer.ci = feval(layer.ci.bpfun,layer.ci,dlossdcest);
+%layer.og = feval(layer.og.fffun,layer.og,inp);
+layer.og = feval(layer.og.bpfun,layer.og,dlossdo);
+%layer.ig = feval(layer.ig.fffun,layer.ig,inp);
+layer.ig = feval(layer.ig.bpfun,layer.ig,dlossdi);
+%layer.fg = feval(layer.fg.fffun,layer.fg,inp);
+layer.fg = feval(layer.fg.bpfun,layer.fg,dlossdf);
+layer.dlossdx=layer.ci.dlossdx(1:layer.nx,1)+...
+    layer.og.dlossdx(1:layer.nx,1)+...
+    layer.ig.dlossdx(1:layer.nx,1)+...
+    layer.fg.dlossdx(1:layer.nx,1);
+layer.dlossdcold=layer.og.dlossdx(1+layer.nx:end,1)+...
+    layer.ig.dlossdx(1+layer.nx:end,1)+...
+    layer.fg.dlossdx(1+layer.nx:end,1);
+end
+
+function layer=lstmresetbp(layer)%########################################
+layer.ci=feval(layer.ci.bpresetfun,layer.ci);
+layer.og=feval(layer.og.bpresetfun,layer.og);
+layer.ig=feval(layer.ci.bpresetfun,layer.ig);
+layer.fg=feval(layer.fg.bpresetfun,layer.fg);
+end
+
+function layer=lstmupdweights(layer,iter,alfa,beta1,beta2,eta,dropout)%#####
+%adam update rules
+layer.fg=feval(layer.fg.updwfun,layer.fg,iter,alfa,beta1,...
+    beta2,eta,dropout);
+layer.ig=feval(layer.ig.updwfun,layer.ig,iter,alfa,beta1,...
+    beta2,eta,dropout);
+layer.og=feval(layer.og.updwfun,layer.og,iter,alfa,beta1,...
+    beta2,eta,dropout);
+layer.ci=feval(layer.ci.updwfun,layer.ci,iter,alfa,beta1,...
+    beta2,eta,dropout);
+end

hybnet/hnet_init_state.m

@@ -0,0 +1,5 @@
+function hnet=hnet_init_state(hnet)%################################
+for i = 1:hnet.nl  
+  hnet.layers{i}=feval(hnet.layers{i}.initstatefun,hnet.layers{i});
+end
+end

hybnet/hnet_reset_bp.m

@@ -0,0 +1,5 @@
+function hnet=hnet_reset_bp(hnet)%################################
+for i = 1:hnet.nl  
+  hnet.layers{i}=feval(hnet.layers{i}.bpresetfun,hnet.layers{i});
+end
+end

hybnet/hnet_update_weights.m

@@ -0,0 +1,10 @@
+function hnet=hnet_update_weights(hnet)%################################
+hnet.iter = hnet.iter+1;
+hnet.alfa=hnet.alfa0-(hnet.alfa0-hnet.alfaf)*hnet.iter/hnet.nter;%hnet.alfa0./sqrt(hnet.iter/50);% Joao Ramos
+for i = 1:hnet.nl
+    if hnet.layers{i}.nw > 0
+        hnet.layers{i}=feval(hnet.layers{i}.updwfun,hnet.layers{i},...
+            hnet.iter,hnet.alfa,hnet.beta1,hnet.beta2,hnet.eta,hnet.dropout);
+    end
+end
+end

hybnet/hnetadam.m

@@ -0,0 +1,48 @@
+function [w,fval]=hnetadam(lossfunc, w0, options, ofun)
+% default adam parameters; in future goes to structure options
+if nargin<3
+    options=[];
+end
+if isfield(options,"adalfa")
+    alfa=  options.adalfa;
+else
+    alfa = 0.001; %AUTOMATE, if undefined, make it n
+end
+beta1=0.9;
+beta2=0.999;
+eta=1e-8;
+if isfield(options,"admini")
+    minibatchsize=options.admini;
+else
+    minibatchsize = 0.8; %AUTOMATE, if undefined, make it n
+end
+if isfield(options,"addrop")
+    dropout=options.addrop;
+else
+    dropout = 0.2; %AUTOMATE, if undefined, make it n
+end
+%initialization of variables
+w=reshape(w0,numel(w0),1); %weights
+m=zeros(numel(w),1);  %first moment vector
+v=zeros(numel(w),1); %Second moment vector
+
+
+for i=1:options.niter  %iteration cylce
+
+   witer=w;
+
+  %call lossfun get residuals and gradients residuals
+  [fval,g]=feval(lossfunc,witer,minibatchsize,dropout);  
+
+   m = beta1*m + (1-beta1)*g;
+   v = beta2*v + (1-beta2)*(g.*g);  
+   mest = m/(1-beta1^i);
+   vest = v/(1-beta2^i);
+   w = w - alfa*(mest./(sqrt(vest)+eta));
+
+end
+[fval]=feval(lossfunc,witer,minibatchsize,dropout);  
+
+
+end
+

hybnet/hnetbp.m

@@ -0,0 +1,17 @@
+function hnet=hnetbp(hnet,dlossdy)%################################
+% hnet.dlossdy =  dlossdy;
+% for i=1:hnet.nl
+%     hnet.layers{i}.dlossdy=zeros(hnet.layers{i}.ny,1);
+% end
+hnet.layers{hnet.nl}.dlossdy = dlossdy;
+for i = hnet.nl:-1:1
+    hnet.layers{i}=feval(hnet.layers{i}.bpfun,hnet.layers{i},hnet.layers{i}.dlossdy);
+    inpl=hnet.layers{i}.inpl;
+    if inpl==0
+        %hnet.dlossdx = hnet.dlossdx + hnet.layers{i}.dlossdx;%???
+    else
+        hnet.layers{inpl}.dlossdy = hnet.layers{inpl}.dlossdy + ...
+            hnet.layers{i}.dlossdx;
+    end
+end
+end

hybnet/hnetcreate.m

@@ -0,0 +1,43 @@
+function hnet=hnetcreate(layers,tag)%######################################
+if nargin<2
+    tag='HNET';
+end
+hnet.tag = tag;
+hnet.layers = layers;
+hnet.nl = numel(layers);
+hnet.nx = layers{1}.nx;
+hnet.ny = layers{hnet.nl}.ny;
+hnet.nw = 0;
+for i=1:hnet.nl
+    hnet.nw = hnet.nw + hnet.layers{i}.nw;
+end
+hnet.setwfun=@hnetsetw;
+hnet.getgradfun=@hnetgetgrad;
+hnet.fffun=@hnetff;
+hnet.bpfun=@hnetbp;
+%adam updae rules
+hnet.iter=0;
+hnet.alfa = 0.001; %AUTOMATE, if undefined, make it n
+hnet.beta1=0.9;
+hnet.beta2=0.999;
+hnet.eta=1e-8;
+hnet.minibatch = 0.8;
+hnet.dropout = 0.1;
+%verify connectivity
+for i=1:hnet.nl
+    if hnet.layers{i}.inpl==-1
+        hnet.layers{i}.inpl=i-1;
+    else
+        ind = hnet.layers{i}.inpl>=i;
+        assert(sum(ind)==0,'layer connectivy error')
+    end
+end
+end
+
+
+
+
+
+
+
+

hybnet/hnetff.m

@@ -0,0 +1,21 @@
+function hnet=hnetff(hnet,inpdata)%########################################
+inpdata = reshape(inpdata,numel(inpdata),1);
+hnet.x = inpdata;
+for i=1:hnet.nl
+    inpl=hnet.layers{i}.inpl;
+    if inpl==0
+        inp = inpdata;
+    else
+        inp = hnet.layers{inpl}.y;
+    end
+    hnet.layers{i}=feval(hnet.layers{i}.fffun,hnet.layers{i},inp);
+end
+hnet.y=hnet.layers{hnet.nl}.y;
+
+% revise in the future
+hnet.dlossdy =  zeros(hnet.ny,1);
+for i=1:hnet.nl
+    hnet.layers{i}.dlossdy=zeros(hnet.layers{i}.ny,1);
+end
+
+end