books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into qwen-tokenizer-fix

Browse Source
This commit is contained in:
casinca
2025-09-16 09:38:45 +02:00
committed by GitHub
parent 3a5ee8cfa1 b6cd0a312f
commit 701b5ad54d
12 changed files with 14 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ch05/07_gpt_to_llama/converting-gpt-to-llama2.ipynb
View File

@@ -435,7 +435,7 @@
" positions = torch.arange(context_length)\n",
"\n",
" # Compute the angles\n",
" angles = positions[:, None] * inv_freq[None, :] # Shape: (context_length, head_dim // 2)\n",
" angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) # Shape: (context_length, head_dim // 2)\n",
"\n",
" # Expand angles to match the head_dim\n",
" angles = torch.cat([angles, angles], dim=1) # Shape: (context_length, head_dim)\n",

2
ch05/07_gpt_to_llama/converting-llama2-to-llama3.ipynb
View File

@@ -310,7 +310,7 @@
" positions = torch.arange(context_length)\n",
"\n",
" # Compute the angles\n",
" angles = positions[:, None] * inv_freq[None, :] # Shape: (context_length, head_dim // 2)\n",
" angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) # Shape: (context_length, head_dim // 2)\n",
"\n",
" # Expand angles to match the head_dim\n",
" angles = torch.cat([angles, angles], dim=1) # Shape: (context_length, head_dim)\n",

2
ch05/07_gpt_to_llama/standalone-llama32.ipynb
View File

@@ -180,7 +180,7 @@
" positions = torch.arange(context_length, dtype=dtype)\n",
"\n",
" # Compute the angles\n",
" angles = positions[:, None] * inv_freq[None, :] # Shape: (context_length, head_dim // 2)\n",
" angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) # Shape: (context_length, head_dim // 2)\n",
"\n",
" # Expand angles to match the head_dim\n",
" angles = torch.cat([angles, angles], dim=1) # Shape: (context_length, head_dim)\n",

2
ch05/11_qwen3/standalone-qwen3-moe-plus-kvcache.ipynb
View File

@@ -275,7 +275,7 @@
" positions = torch.arange(context_length, dtype=dtype)\n",
"\n",
" # Compute the angles\n",
" angles = positions[:, None] * inv_freq[None, :] # Shape: (context_length, head_dim // 2)\n",
" angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) # Shape: (context_length, head_dim // 2)\n",
"\n",
" # Expand angles to match the head_dim\n",
" angles = torch.cat([angles, angles], dim=1) # Shape: (context_length, head_dim)\n",

2
ch05/11_qwen3/standalone-qwen3-moe.ipynb
View File

@@ -275,7 +275,7 @@
" positions = torch.arange(context_length, dtype=dtype)\n",
"\n",
" # Compute the angles\n",
" angles = positions[:, None] * inv_freq[None, :] # Shape: (context_length, head_dim // 2)\n",
" angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) # Shape: (context_length, head_dim // 2)\n",
"\n",
" # Expand angles to match the head_dim\n",
" angles = torch.cat([angles, angles], dim=1) # Shape: (context_length, head_dim)\n",

2
ch05/11_qwen3/standalone-qwen3-plus-kvcache.ipynb
View File

@@ -206,7 +206,7 @@
" positions = torch.arange(context_length, dtype=dtype)\n",
"\n",
" # Compute the angles\n",
" angles = positions[:, None] * inv_freq[None, :] # Shape: (context_length, head_dim // 2)\n",
" angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) # Shape: (context_length, head_dim // 2)\n",
"\n",
" # Expand angles to match the head_dim\n",
" angles = torch.cat([angles, angles], dim=1) # Shape: (context_length, head_dim)\n",

2
ch05/11_qwen3/standalone-qwen3.ipynb
View File

@@ -204,7 +204,7 @@
" positions = torch.arange(context_length, dtype=dtype)\n",
"\n",
" # Compute the angles\n",
" angles = positions[:, None] * inv_freq[None, :] # Shape: (context_length, head_dim // 2)\n",
" angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) # Shape: (context_length, head_dim // 2)\n",
"\n",
" # Expand angles to match the head_dim\n",
" angles = torch.cat([angles, angles], dim=1) # Shape: (context_length, head_dim)\n",

2
ch05/12_gemma3/standalone-gemma3-plus-kvcache.ipynb
View File

@@ -200,7 +200,7 @@
" positions = torch.arange(context_length, dtype=dtype)\n",
"\n",
" # Compute the angles\n",
" angles = positions[:, None] * inv_freq[None, :] # Shape: (context_length, head_dim // 2)\n",
" angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) # Shape: (context_length, head_dim // 2)\n",
"\n",
" # Expand angles to match the head_dim\n",
" angles = torch.cat([angles, angles], dim=1) # Shape: (context_length, head_dim)\n",

2
ch05/12_gemma3/standalone-gemma3.ipynb
View File

@@ -200,7 +200,7 @@
" positions = torch.arange(context_length, dtype=dtype)\n",
"\n",
" # Compute the angles\n",
" angles = positions[:, None] * inv_freq[None, :] # Shape: (context_length, head_dim // 2)\n",
" angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) # Shape: (context_length, head_dim // 2)\n",
"\n",
" # Expand angles to match the head_dim\n",
" angles = torch.cat([angles, angles], dim=1) # Shape: (context_length, head_dim)\n",

6
ch06/03_bonus_imdb-classification/train_sklearn_logreg.py
View File

@@ -19,7 +19,7 @@ def load_dataframes():
return df_train, df_val, df_test
def eval(model, X_train, y_train, X_val, y_val, X_test, y_test):
def eval_model(model, X_train, y_train, X_val, y_val, X_test, y_test):
# Making predictions
y_pred_train = model.predict(X_train)
y_pred_val = model.predict(X_val)
@@ -67,9 +67,9 @@ if __name__ == "__main__":
dummy_clf.fit(X_train, y_train)
print("Dummy classifier:")
eval(dummy_clf, X_train, y_train, X_val, y_val, X_test, y_test)
eval_model(dummy_clf, X_train, y_train, X_val, y_val, X_test, y_test)
print("\n\nLogistic regression classifier:")
model = LogisticRegression(max_iter=1000)
model.fit(X_train, y_train)
eval(model, X_train, y_train, X_val, y_val, X_test, y_test)
eval_model(model, X_train, y_train, X_val, y_val, X_test, y_test)

2
pkg/llms_from_scratch/llama3.py
View File

@@ -238,7 +238,7 @@ def compute_rope_params(head_dim, theta_base=10_000, context_length=4096, freq_c
positions = torch.arange(context_length, dtype=dtype)
# Compute the angles
angles = positions[:, None] * inv_freq[None, :] # Shape: (context_length, head_dim // 2)
angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) # Shape: (context_length, head_dim // 2)
# Expand angles to match the head_dim
angles = torch.cat([angles, angles], dim=1) # Shape: (context_length, head_dim)

2
pkg/llms_from_scratch/qwen3.py
View File

@@ -326,7 +326,7 @@ def compute_rope_params(head_dim, theta_base=10_000, context_length=4096, dtype=
positions = torch.arange(context_length, dtype=dtype)
# Compute the angles
angles = positions[:, None] * inv_freq[None, :] # Shape: (context_length, head_dim // 2)
angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) # Shape: (context_length, head_dim // 2)
# Expand angles to match the head_dim
angles = torch.cat([angles, angles], dim=1) # Shape: (context_length, head_dim)