diff --git a/__pycache__/ffnn.cpython-39.pyc b/__pycache__/ffnn.cpython-39.pyc new file mode 100644 index 0000000..760ce83 Binary files /dev/null and b/__pycache__/ffnn.cpython-39.pyc differ diff --git a/__pycache__/layer.cpython-39.pyc b/__pycache__/layer.cpython-39.pyc new file mode 100644 index 0000000..ad17f6a Binary files /dev/null and b/__pycache__/layer.cpython-39.pyc differ diff --git a/__pycache__/neuron.cpython-39.pyc b/__pycache__/neuron.cpython-39.pyc new file mode 100644 index 0000000..408b644 Binary files /dev/null and b/__pycache__/neuron.cpython-39.pyc differ diff --git a/__pycache__/scalar.cpython-39.pyc b/__pycache__/scalar.cpython-39.pyc new file mode 100644 index 0000000..1a984a1 Binary files /dev/null and b/__pycache__/scalar.cpython-39.pyc differ diff --git a/__pycache__/vis.cpython-39.pyc b/__pycache__/vis.cpython-39.pyc new file mode 100644 index 0000000..047fc24 Binary files /dev/null and b/__pycache__/vis.cpython-39.pyc differ diff --git a/neuron.py b/neuron.py index a0e66ec..a2608e2 100644 --- a/neuron.py +++ b/neuron.py @@ -4,17 +4,16 @@ class Neuron: - def __init__(self,nin): + def __init__(self,nin): + self.w = [Scalar(random.uniform(-1,1)) for i in range(nin)] + self.b = Scalar(random.uniform(-1,1)) - self.w = [Scalar(random.uniform(-1,1)) for i in range(nin)] - self.b = random.uniform(-1,1) + def __call__(self,x): + activation = sum((wj*xj for wj,xj in zip(self.w,x)),self.b) + output = activation.tanh() + return output - def __call__(self,x): + def parameters(self): - out = sum(w_i*x_i for w_i,x_i in zip(self.w,x)) - activation = (out+self.b).tanh() - return activation - - def parameters(self): - - return self.w + [self.b] + return self.w + [self.b] + diff --git a/scalar.py b/scalar.py index 82fef0d..570271f 100644 --- a/scalar.py +++ b/scalar.py @@ -1,7 +1,8 @@ from vis import * +import numpy as np class Scalar: - + def __init__(self,data,_children=(),_op='',label=''): self.data = data self.grad = 0.0 @@ -9,93 +10,97 @@ def __init__(self,data,_children=(),_op='',label=''): self._prev = set(_children) self._op = _op self.label = label - + def __repr__(self): return f"Scalar(data={self.data})" - + def __add__(self,other): - - + + other = other if isinstance(other,Scalar) else Scalar(other) out = Scalar(self.data+other.data,(self,other),'+') - + def _backward(): - - self.grad += 1.0*out.grad - other.grad += 1.0*out.grad - + + self.grad += out.grad + other.grad += out.grad + out._backward = _backward - + return out - + def __mul__(self,other): - + other = other if isinstance(other,Scalar) else Scalar(other) out = Scalar(self.data*other.data,(self,other),'*') - + def _backward(): - + self.grad += other.data*out.grad other.grad += self.data*out.grad - + out._backward = _backward return out - + def __rmul__(self,other): return self*other - + def exp(self): out = Scalar(math.exp(self.data),(self,),'exp') - + def _backward(): self.backward += out.data*out.grad out._backward = _backward - + return out - + def __pow__(self,other): - + assert isinstance(other,(int,float)),'support float/int' out = Scalar(self.data**other,(self,),f'**{other}') - + def _backward(): self.grad += other*(self.data**(other-1))*out.grad out._backward = _backward - + return out - + def __truediv__(self,other): return self*other**-1 - + def __neg__(self): return self*-1 - + def __sub__(self,other): return self + (-other) - + + def __radd__(self, other): # other + self + return self + other + def tanh(self): n = self.data - ot = (math.exp(2*n)-1)/(math.exp(2*n)+1) - + ot = (np.exp(2*n)-1)/(np.exp(2*n)+1) + out = Scalar(ot,(self,),'tanh') - + def _backward(): self.grad += (1 - ot**2)*out.grad out._backward = _backward return out - + def backward(self): topo = [] visited = set() def build_topo(v): - if v not in visited: - visited.add(v) - for child in v._prev: - build_topo(child) - topo.append(v) + if v not in visited: + visited.add(v) + for child in v._prev: + build_topo(child) + topo.append(v) build_topo(self) - self.grad = 1.0 - for node in reversed(topo): - node._backward() + # go one variable at a time and apply the chain rule to get its gradient + self.grad = 1 + for v in reversed(topo): + v._backward() diff --git a/test.py b/test.py new file mode 100644 index 0000000..c86c2d9 --- /dev/null +++ b/test.py @@ -0,0 +1,45 @@ +from ffnn import * +import matplotlib.pyplot as plt + + +f = FFNN(3,[4,3,1]) + +test_data = [[random.uniform(-1,1) for i in range(3)] for _ in range(10)] +labels = [random.choice([0,1]) for i in range(len(test_data))] + +pred = [f(data) for data in test_data] + +#implementing Stochastic Gradient Descent + +epochs = 10 +eta = 0.01 + +losses = [] + +for i in range(epochs): + + print("---------------Epoch -------------------",i) + + # Forward propagation + pred = [f(data) for data in test_data] + loss = sum((y_pred-y_true)**2 for y_pred,y_true in zip(pred,labels)) + losses.append(loss.data) + + print('Loss ',i,loss) + + for parameter in f.parameters(): + parameter.grad = 0.0 + + # Backward propagation + + loss.backward() + + # Updating parameters + + for parameter in f.parameters(): + parameter.data += -eta * parameter.grad + + +plt.plot([i for i in range(epochs)],losses) +plt.xlabel('epoch') +plt.ylabel('loss')