books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

mha variants

Browse Source
This commit is contained in:
rasbt
2024-03-06 08:30:32 -06:00
parent d4754f1bdd
commit 87fcfd9245
10 changed files with 431 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ch04/01_main-chapter-code/gpt.py
View File

@@ -56,7 +56,7 @@ def create_dataloader_v1(txt, batch_size=4, max_length=256,
class MultiHeadAttention(nn.Module):
def __init__(self, d_in, d_out, block_size, dropout, num_heads, qkv_bias=False):
super().__init__()
assert d_out % num_heads == 0, "d_out must be divisible by n_heads"
assert d_out % num_heads == 0, "d_out must be divisible by num_heads"
self.d_out = d_out
self.num_heads = num_heads

2
ch04/01_main-chapter-code/previous_chapters.py
View File

@@ -45,7 +45,7 @@ def create_dataloader_v1(txt, batch_size=4, max_length=256,
class MultiHeadAttention(nn.Module):
def __init__(self, d_in, d_out, block_size, dropout, num_heads, qkv_bias=False):
super().__init__()
assert d_out % num_heads == 0, "d_out must be divisible by n_heads"
assert d_out % num_heads == 0, "d_out must be divisible by num_heads"
self.d_out = d_out
self.num_heads = num_heads