Skip to content

Torch compilable RoPE #51

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
NickCheng0921 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Torch compilable RoPE #51

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 5 additions & 18 deletions llada/model/modeling_llada.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -398,29 +398,16 @@ def __init__(self, config: ModelConfig, cache: BufferCache):
self.get_rotary_embedding(config.max_sequence_length, _non_meta_init_device(config))

def get_rotary_embedding(self, seq_len: int, device: torch.device) -> Tuple[torch.Tensor, torch.Tensor]:
if (
(pos_sin := self.__cache.get("rope_pos_sin")) is not None
and (pos_cos := self.__cache.get("rope_pos_cos")) is not None
and pos_sin.shape[-2] >= seq_len
and pos_cos.shape[-2] >= seq_len
):
if pos_sin.device != device:
pos_sin = pos_sin.to(device)
self.__cache["rope_pos_sin"] = pos_sin
if pos_cos.device != device:
pos_cos = pos_cos.to(device)
self.__cache["rope_pos_cos"] = pos_cos
return pos_sin[:, :, :seq_len, :], pos_cos[:, :, :seq_len, :]

with torch.autocast(device.type, enabled=False):
dim = self.config.d_model // self.config.n_heads
inv_freq = 1.0 / (self.rope_theta ** (torch.arange(0, dim, 2, device=device, dtype=torch.float) / dim))
seq = torch.arange(seq_len, device=device, dtype=torch.float)
freqs = einsum("i , j -> i j", seq, inv_freq)
Copy link
Author

@NickCheng0921 NickCheng0921 Dec 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

using the torch outer product here instead of einsum gives 3% faster forward() calls from my testing
doesn't make a big difference though

positions = torch.cat((freqs, freqs), dim=-1)
pos_sin, pos_cos = positions.sin()[None, None, :, :], positions.cos()[None, None, :, :]
self.__cache["rope_pos_sin"] = pos_sin
self.__cache["rope_pos_cos"] = pos_cos
positions = torch.cat([freqs, freqs], dim=-1)

# unsqueeze over cache allows torch.compile
pos_sin = torch.sin(positions).unsqueeze(0).unsqueeze(0)
pos_cos = torch.cos(positions).unsqueeze(0).unsqueeze(0)
return pos_sin, pos_cos

def rotate_half(self, x: torch.Tensor) -> torch.Tensor:
Expand Down