books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add LoRA scaling (#823)

Browse Source
This commit is contained in:
Sebastian Raschka
2025-09-14 11:57:55 -05:00
committed by GitHub
parent fc101b710e
commit 8f3e5b024d
2 changed files with 44 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
appendix-E/01_main-chapter-code/appendix-E.ipynb
View File

File diff suppressed because one or more lines are too long

4
pkg/llms_from_scratch/appendix_e.py
View File

@@ -14,9 +14,11 @@ class LoRALayer(torch.nn.Module):
torch.nn.init.kaiming_uniform_(self.A, a=math.sqrt(5)) # similar to standard weight initialization torch.nn.init.kaiming_uniform_(self.A, a=math.sqrt(5)) # similar to standard weight initialization
self.B = torch.nn.Parameter(torch.zeros(rank, out_dim)) self.B = torch.nn.Parameter(torch.zeros(rank, out_dim))
self.alpha = alpha self.alpha = alpha
self.rank = rank
def forward(self, x): def forward(self, x):
x = self.alpha * (x @ self.A @ self.B)
x = (self.alpha / self.rank) * (x @ self.A @ self.B)
return x return x