add dropout for embedding layers

2026-04-10 12:33:42 +00:00 · 2024-03-04 07:05:06 -06:00
parent 3198363c4f
commit e0df4df433
4 changed files with 66 additions and 24 deletions
--- a/ch04/01_main-chapter-code/gpt.py
+++ b/ch04/01_main-chapter-code/gpt.py
@@ -202,6 +202,7 @@ class GPTModel(nn.Module):
        tok_embeds = self.tok_emb(in_idx)
        pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device))
        x = tok_embeds + pos_embeds  # Shape [batch_size, num_tokens, emb_size]
+        x = self.drop_emb(x)
        x = self.trf_blocks(x)
        x = self.final_norm(x)
        logits = self.out_head(x)