Skip to content

Commit

Permalink
Fix dumb part of test failures
Browse files Browse the repository at this point in the history
  • Loading branch information
norabelrose committed Mar 25, 2023
1 parent 5dc2ec6 commit b1b95e5
Showing 1 changed file with 2 additions and 19 deletions.
21 changes: 2 additions & 19 deletions tests/test_smoke_elicit.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,14 @@
from elk.training import CcsReporterConfig, EigenReporterConfig
from elk.training.train import train, RunConfig

"""
TODO: These tests should work with deberta
but you'll need to make deberta fp32 instead of fp16
because pytorch cpu doesn't support fp16
"""


def test_smoke_elicit_run_tiny_gpt2_ccs(tmp_path: Path):
model_path = "sshleifer/tiny-gpt2"
dataset_name = "imdb"
config = RunConfig(
data=ExtractionConfig(
model=model_path,
prompts=PromptConfig(dataset=dataset_name, max_examples=[10]),
prompts=PromptConfig(datasets=[dataset_name], max_examples=[10]),
# run on all layers, tiny-gpt only has 2 layers
),
net=CcsReporterConfig(),
Expand All @@ -33,23 +27,12 @@ def test_smoke_elicit_run_tiny_gpt2_ccs(tmp_path: Path):


def test_smoke_elicit_run_tiny_gpt2_eigen(tmp_path: Path):
"""
Currently this test fails with
u -= torch.einsum("...ij,...i->...j", V[..., :k, :], proj)
V[..., k, :] = F.normalize(u, dim=-1)
~~~~~~~~~ <--- HERE
u[:] = torch.einsum("...ij,...j->...i", A, V[..., k, :])
RuntimeError: select(): index 1 out of range for tensor of size [1, 2]
at dimension 0
"""
model_path = "sshleifer/tiny-gpt2"
dataset_name = "imdb"
config = RunConfig(
data=ExtractionConfig(
model=model_path,
prompts=PromptConfig(dataset=dataset_name, max_examples=[10]),
prompts=PromptConfig(datasets=[dataset_name], max_examples=[10]),
# run on all layers, tiny-gpt only has 2 layers
),
net=EigenReporterConfig(),
Expand Down

0 comments on commit b1b95e5

Please sign in to comment.