books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

ch4 exercise solutions

Browse Source
This commit is contained in:
rasbt
2024-02-11 11:51:39 -06:00
parent 103f7826ad
commit fe332006de
4 changed files with 394 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
ch04/01_main-chapter-code/gpt.py
View File

@@ -187,12 +187,11 @@ class GPTModel(nn.Module):
super().__init__()
self.tok_emb = nn.Embedding(cfg["vocab_size"], cfg["emb_dim"])
self.pos_emb = nn.Embedding(cfg["ctx_len"], cfg["emb_dim"])
self.drop_emb = nn.Dropout(cfg["drop_rate"])
# Use a placeholder for TransformerBlock
self.trf_blocks = nn.Sequential(
*[TransformerBlock(cfg) for _ in range(cfg["n_layers"])])
# Use a placeholder for LayerNorm
self.final_norm = LayerNorm(cfg["emb_dim"])
self.out_head = nn.Linear(cfg["emb_dim"], cfg["vocab_size"], bias=False)