from abc import ABC, abstractmethod\n",
"\n",
"class Module(ABC):\n",
" def __init__(self):\n",
" self._parameters = []\n",
"\n",
" @final\n",
" def parameters(self) -> list:\n",
" return self._parameters\n",
"\n",
" @abstractmethod\n",
" def __call__(self, x: list):\n",
" pass\n",
"\n",
" @final\n",
" def zero_grad(self):\n",
" for p in self.parameters():\n",
" p.grad = 0\n",
"class Neuron(Module):\n",
" def __init__(self, n_in, activation=None):\n",
" self.n_in = n_in\n",
" self.act = activation\n",
"\n",
" self.w = [Node(random.random()) for _ in range(n_in)]\n",
" self.b = Node(0.0)\n",
" self._parameters = self.w + [self.b]\n",
"\n",
" def __call__(self, x: list):\n",
" assert len(x) == self.n_in\n",
" out = sum((x[j] * self.w[j] for j in range(self.n_in)), start=self.b)\n",
" if self.act is not None:\n",
" if self.act == "tanh":\n",
" out = out.tanh()\n",
" elif self.act == "relu":\n",
" out = out.relu()\n",
" else:\n",
" raise NotImplementedError("Activation not supported.")\n",
" return out\n",
"\n",
" def __repr__(self):\n",
" return f"{self.act if self.act is not None else 'linear'}({len(self.w)})"\n",
"\n",
"\n",
"class Layer(Module):\n",
" def __init__(self, n_in, n_out, *args):\n",
" self.neurons = [Neuron(n_in, *args) for _ in range(n_out)]\n",
" self._parameters = [p for n in self.neurons for p in n.parameters()]\n",
"\n",
" def __call__(self, x: list):\n",
" out = [n(x) for n in self.neurons]\n",
" return out[0] if len(out) == 1 else out\n",
"\n",
" def __repr__(self):\n",
" return f"Layer[{', '.join(str(n) for n in self.neurons)}]"\n",
"\n",
"\n",
"class MLP(Module):\n",
" def __init__(self, n_in, n_outs, activation=None):\n",
" sizes = [n_in] + n_outs\n",
" self.layers = []\n",
" for i in range(len(n_outs)):\n",
" act = activation if i < len(n_outs) - 1 else None\n",
" layer = Layer(sizes[i], sizes[i + 1], act)\n",
" self.layers.append(layer)\n",
"\n",
" self._parameters = [p for layer in self.layers for p in layer.parameters()]\n",
"\n",
" def __call__(self, x):\n",
" for layer in self.layers:\n",
" x = layer(x)\n",
" return x\n",
"\n",
" def __repr__(self):\n",
" return f"MLP[{', '.join(str(layer) for layer in self.layers)}]"\n",
"