mirror of
https://github.com/rasbt/LLMs-from-scratch.git
synced 2026-04-10 12:33:42 +00:00
Make quote style consistent (#891)
This commit is contained in:
committed by
GitHub
parent
9276edbc37
commit
7ca7c47e4a
@@ -73,7 +73,7 @@ class MultiHeadAttention(nn.Module):
|
||||
self.W_value = nn.Linear(d_in, d_out, bias=qkv_bias)
|
||||
self.out_proj = nn.Linear(d_out, d_out) # Linear layer to combine head outputs
|
||||
self.dropout = nn.Dropout(dropout)
|
||||
self.register_buffer('mask', torch.triu(torch.ones(context_length, context_length), diagonal=1))
|
||||
self.register_buffer("mask", torch.triu(torch.ones(context_length, context_length), diagonal=1))
|
||||
|
||||
def forward(self, x):
|
||||
b, num_tokens, d_in = x.shape
|
||||
@@ -250,8 +250,8 @@ def assign(left, right):
|
||||
|
||||
|
||||
def load_weights_into_gpt(gpt, params):
|
||||
gpt.pos_emb.weight = assign(gpt.pos_emb.weight, params['wpe'])
|
||||
gpt.tok_emb.weight = assign(gpt.tok_emb.weight, params['wte'])
|
||||
gpt.pos_emb.weight = assign(gpt.pos_emb.weight, params["wpe"])
|
||||
gpt.tok_emb.weight = assign(gpt.tok_emb.weight, params["wte"])
|
||||
|
||||
for b in range(len(params["blocks"])):
|
||||
q_w, k_w, v_w = np.split(
|
||||
@@ -311,7 +311,7 @@ def load_weights_into_gpt(gpt, params):
|
||||
|
||||
|
||||
def text_to_token_ids(text, tokenizer):
|
||||
encoded = tokenizer.encode(text, allowed_special={'<|endoftext|>'})
|
||||
encoded = tokenizer.encode(text, allowed_special={"<|endoftext|>"})
|
||||
encoded_tensor = torch.tensor(encoded).unsqueeze(0) # add batch dimension
|
||||
return encoded_tensor
|
||||
|
||||
|
||||
@@ -261,7 +261,7 @@ if __name__ == "__main__":
|
||||
)
|
||||
parser.add_argument(
|
||||
"--average_embeddings",
|
||||
action='store_true',
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Average the output embeddings from all tokens instead of using"
|
||||
|
||||
Reference in New Issue
Block a user