add dropout for embedding layers

2026-04-10 12:33:42 +00:00 · 2024-03-04 07:05:06 -06:00
parent 3198363c4f
commit e0df4df433
4 changed files with 66 additions and 24 deletions
--- a/ch04/01_main-chapter-code/exercise-solutions.ipynb
+++ b/ch04/01_main-chapter-code/exercise-solutions.ipynb
@@ -336,6 +336,7 @@
    "        tok_embeds = self.tok_emb(in_idx)\n",
    "        pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device))\n",
    "        x = tok_embeds + pos_embeds  # Shape [batch_size, num_tokens, emb_size]\n",
+    "        x = self.drop_emb(x)\n",
    "        x = self.trf_blocks(x)\n",
    "        x = self.final_norm(x)\n",
    "        logits = self.out_head(x)\n",