books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make quote style consistent (#891)

Browse Source
This commit is contained in:
Sebastian Raschka
2025-10-21 19:42:33 -05:00
committed by GitHub
parent 9276edbc37
commit 7ca7c47e4a
24 changed files with 239 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ch05/01_main-chapter-code/previous_chapters.py
View File

@@ -71,7 +71,7 @@ class MultiHeadAttention(nn.Module):
self.W_value = nn.Linear(d_in, d_out, bias=qkv_bias)
self.out_proj = nn.Linear(d_out, d_out) # Linear layer to combine head outputs
self.dropout = nn.Dropout(dropout)
self.register_buffer('mask', torch.triu(torch.ones(context_length, context_length), diagonal=1))
self.register_buffer("mask", torch.triu(torch.ones(context_length, context_length), diagonal=1))
def forward(self, x):
b, num_tokens, d_in = x.shape

2
ch05/03_bonus_pretraining_on_gutenberg/prepare_dataset.py
View File

@@ -43,7 +43,7 @@ def combine_files(file_paths, target_dir, max_size_mb=500, separator="<|endoftex
content = strip_headers(content)
# Regular expression to replace multiple blank lines with a single blank line
content = re.sub(r'\n\s*\n', '\n\n', content)
content = re.sub(r"\n\s*\n", "\n\n", content)
estimated_size = len(content.encode("utf-8"))
if current_size + estimated_size > max_size_mb * 1024 * 1024:

38
ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py
View File

@@ -148,26 +148,26 @@ def train_model_simple(model, optimizer, device, n_epochs,
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='GPT Model Training Configuration')
parser = argparse.ArgumentParser(description="GPT Model Training Configuration")
parser.add_argument('--data_dir', type=str, default='gutenberg/data',
help='Directory containing the training data')
parser.add_argument('--output_dir', type=str, default='model_checkpoints',
help='Directory where the model checkpoints will be saved')
parser.add_argument('--n_epochs', type=int, default=1,
help='Number of epochs to train the model')
parser.add_argument('--print_sample_iter', type=int, default=1000,
help='Iterations between printing sample outputs')
parser.add_argument('--eval_freq', type=int, default=100,
help='Frequency of evaluations during training')
parser.add_argument('--save_ckpt_freq', type=int, default=100_000,
help='Frequency of saving model checkpoints during training')
parser.add_argument('--lr', type=float, default=5e-4,
help='Learning rate for the optimizer')
parser.add_argument('--batch_size', type=int, default=4,
help='Batch size for training')
parser.add_argument('--debug', type=bool, default=False,
help='Uses a very small model for debugging purposes')
parser.add_argument("--data_dir", type=str, default="gutenberg/data",
help="Directory containing the training data")
parser.add_argument("--output_dir", type=str, default="model_checkpoints",
help="Directory where the model checkpoints will be saved")
parser.add_argument("--n_epochs", type=int, default=1,
help="Number of epochs to train the model")
parser.add_argument("--print_sample_iter", type=int, default=1000,
help="Iterations between printing sample outputs")
parser.add_argument("--eval_freq", type=int, default=100,
help="Frequency of evaluations during training")
parser.add_argument("--save_ckpt_freq", type=int, default=100_000,
help="Frequency of saving model checkpoints during training")
parser.add_argument("--lr", type=float, default=5e-4,
help="Learning rate for the optimizer")
parser.add_argument("--batch_size", type=int, default=4,
help="Batch size for training")
parser.add_argument("--debug", type=bool, default=False,
help="Uses a very small model for debugging purposes")
args = parser.parse_args()

2
ch05/05_bonus_hparam_tuning/hparam_search.py
View File

@@ -118,7 +118,7 @@ if __name__ == "__main__":
print(f"Total hyperparameter configurations: {total_combinations}")
# Placeholder for the best loss and best hyperparameters
best_val_loss = float('inf')
best_val_loss = float("inf")
best_hparams = {}
script_path = os.path.abspath(__file__)

2
ch05/07_gpt_to_llama/previous_chapters.py
View File

@@ -38,7 +38,7 @@ def generate(model, idx, max_new_tokens, context_size, temperature=0.0, top_k=No
# Keep only top_k values
top_logits, _ = torch.topk(logits, top_k)
min_val = top_logits[:, -1]
logits = torch.where(logits < min_val, torch.tensor(float('-inf')).to(logits.device), logits)
logits = torch.where(logits < min_val, torch.tensor(float("-inf")).to(logits.device), logits)
# New: Apply temperature scaling
if temperature > 0.0:

2
ch05/08_memory_efficient_weight_loading/previous_chapters.py
View File

@@ -29,7 +29,7 @@ class MultiHeadAttention(nn.Module):
self.W_value = nn.Linear(d_in, d_out, bias=qkv_bias)
self.out_proj = nn.Linear(d_out, d_out) # Linear layer to combine head outputs
self.dropout = nn.Dropout(dropout)
self.register_buffer('mask', torch.triu(torch.ones(context_length, context_length), diagonal=1))
self.register_buffer("mask", torch.triu(torch.ones(context_length, context_length), diagonal=1))
def forward(self, x):
b, num_tokens, d_in = x.shape

2
ch05/10_llm-training-speed/00_orig.py
View File

@@ -426,7 +426,7 @@ def main(gpt_config, settings):
if not os.path.exists(file_path):
with urllib.request.urlopen(url) as response:
text_data = response.read().decode('utf-8')
text_data = response.read().decode("utf-8")
with open(file_path, "w", encoding="utf-8") as file:
file.write(text_data)
else: