wassname/discovering_latent_knowledge
1
0
Fork 0
mirror of https://github.com/wassname/discovering_latent_knowledge.git synced 2026-09-09 11:21:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

dataset refactored but there is a 🐛

Browse Source
This commit is contained in:
deep1
2023-08-05 20:00:15 +08:00
parent 8cb669f121
commit d15b4fa54d
5 changed files with 838 additions and 654 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mjc_notes.md
+3 -2
View File
@@ -859,9 +859,10 @@ TODO
- [ ] do checks
- [ ] for high prob
- [ ] and acc
- [ ] name ds
- [ ] save ds
- [x] name ds
- [x] save ds
- [ ] get model nb working
- [ ] round up the FIXME TODO UPTO HACK's
Got unsupported ScalarType BFloat16
notebooks/03_make_dataset.ipynb
+791 -561
View File
File diff suppressed because it is too large Load Diff
src/datasets/batch.py
+41 -10
View File
@@ -3,9 +3,11 @@ from tqdm.auto import tqdm
from src.datasets.hs import ExtractHiddenStates
from torch.utils.data import DataLoader
from datasets import Dataset
import hashlib
import pickle
import numpy as np
def batch_hidden_states(ehs: ExtractHiddenStates, data: Dataset, n=100, batch_size=2, mcdropout=True):
def batch_hidden_states(model, tokenizer, data: Dataset, n=100, batch_size=2, mcdropout=True):
"""
Given an encoder-decoder model, a list of data, computes the contrast hidden states on n random examples.
Returns numpy arrays of shape (n, hidden_dim) for each candidate label, along with a boolean numpy array of shape (n,)
@@ -13,6 +15,7 @@ def batch_hidden_states(ehs: ExtractHiddenStates, data: Dataset, n=100, batch_si
This is deliberately simple so that it's easy to understand, rather than being optimized for efficiency
"""
ehs = ExtractHiddenStates(model, tokenizer)
ds_t_subset = data.select(range(n))
ds_t_subset.set_format(type='torch', columns=['input_ids', 'label'])
@@ -27,21 +30,21 @@ def batch_hidden_states(ehs: ExtractHiddenStates, data: Dataset, n=100, batch_si
index = i*batch_size+np.arange(nn)
# different due to dropout
hs1 = ehs.get_batch_of_hidden_states(input_ids=input_ids, use_mcdropout=mcdropout)
hs0 = ehs.get_batch_of_hidden_states(input_ids=input_ids, use_mcdropout=mcdropout)
if mcdropout:
hs2 = ehs.get_batch_of_hidden_states(input_ids=input_ids, use_mcdropout=mcdropout)
hs1 = ehs.get_batch_of_hidden_states(input_ids=input_ids, use_mcdropout=mcdropout)
# QC
if i==0:
eps=1e-5
mpe = lambda x,y: np.mean(np.abs(x-y)/(np.abs(x)+np.abs(y)+eps))
a,b=hs2['hidden_states'],hs1['hidden_states']
a,b=hs1['hidden_states'],hs0['hidden_states']
assert mpe(a,b)>eps, "the hidden state pairs should be different but are not. Check model.config.use_cache==False, check this model has dropout in it's arch"
# FIXME, move check to loading?
# assert ((hs1['prob_y']+hs1['prob_n'])>0.5).all(), "your chosen binary answers should take up a lot of the prob space, otherwise choose differen't tokens"
# assert ((hs0['prob_y']+hs0['prob_n'])>0.5).all(), "your chosen binary answers should take up a lot of the prob space, otherwise choose differen't tokens"
else:
hs2 = hs1
hs1 = hs0
for j in range(nn):
@@ -50,14 +53,42 @@ def batch_hidden_states(ehs: ExtractHiddenStates, data: Dataset, n=100, batch_si
info = ds_p_subset[k]
yield dict(
hs1=hs1['hidden_states'][j],
scores1=hs1["scores"][j],
hs0=hs0['hidden_states'][j],
scores1=hs0["scores"][j],
hs2=hs2['hidden_states'][j],
scores2=hs2["scores"][j],
hs1=hs1['hidden_states'][j],
scores2=hs1["scores"][j],
true=true_labels[j].item(),
index=index[j],
**info
)
def md5hash(s: bytes) -> str:
return hashlib.md5(s).hexdigest()
# unique hash
def get_unique_config_name(prompt_fn, model, tokenizer, data, N):
"""
generates a unique name
datasets would do this use the generation kwargs but this way we have control and can handle non-picklable models and thing like the output of prompt functions if they change
# """
example_prompt1 = prompt_fn("text", response=0, lie=True)
model_repo = model.config._name_or_path
kwargs = [str(model), str(tokenizer), str(data), str(prompt_fn.__name__), N]
key = pickle.dumps(kwargs, 1)
hsh = md5hash(key)[:6]
sanitize = lambda s:s.replace('/', '').replace('-', '_') if s is not None else s
config_name = f"{sanitize(model_repo)}-N_{N}-ns-{hsh}"
info_kwargs = dict(model_repo=model_repo, config=model.config, data=str(data), prompt_fn=str(prompt_fn.__name__), N=N,
example_prompt1=example_prompt1,
config_name=config_name)
return config_name, info_kwargs
src/datasets/hs.py
-80
View File
@@ -61,7 +61,6 @@ class ExtractHiddenStates:
input_text: Optional[List[str]] = None,
input_ids: torch.Tensor = None,
truncation_length=999,
output_attentions=False,
use_mcdropout=True,
debug=False,
):
@@ -92,7 +91,6 @@ class ExtractHiddenStates:
outputs = self.model.forward(
input_ids,
return_dict=True,
output_attentions=output_attentions,
output_hidden_states=True,
use_cache=False,
)
@@ -101,12 +99,6 @@ class ExtractHiddenStates:
layers = self.get_layer_selection(outputs)
attentions = None
if output_attentions:
attentions = [outputs["attentions"][i][:, -1] for i in layers]
attentions = torch.stack(attentions, 1)
# shape is [(batch_size, num_heads, input_length, input_length)]*num_layers
hidden_states = torch.stack(
[outputs["hidden_states"][i] for i in layers], 1
)
@@ -117,7 +109,6 @@ class ExtractHiddenStates:
out = dict(
hidden_states=hidden_states,
attentions=attentions,
scores=outputs["scores"],
input_ids=input_ids,
)
@@ -141,74 +132,3 @@ class ExtractHiddenStates:
self.layer_stride,
)
def batch_hidden_states(self, data: Dataset, n=100, batch_size=2, mcdropout=True):
"""
Given an encoder-decoder model, a list of data, computes the contrast hidden states on n random examples.
Returns numpy arrays of shape (n, hidden_dim) for each candidate label, along with a boolean numpy array of shape (n,)
with the ground truth labels
This is deliberately simple so that it's easy to understand, rather than being optimized for efficiency
"""
ds_t_subset = data.select(range(n))
ds_t_subset.set_format(type='torch', columns=['input_ids', 'label'])
ds_p_subset = data.select(range(n))
ds_p_subset.set_format(type="pandas", columns=['lie', 'label', 'prompt', 'prompt_truncated'])
dl = DataLoader(ds_t_subset, batch_size=batch_size, shuffle=True)
for i, batch in enumerate(tqdm(dl, desc='get hidden states')):
input_ids, true_labels = batch["input_ids"], batch["label"]
nn = len(input_ids)
index = i*batch_size+np.arange(nn)
# different due to dropout
hs1 = self.get_batch_of_hidden_states(input_ids=input_ids, use_mcdropout=mcdropout)
if mcdropout:
hs2 = self.get_batch_of_hidden_states(input_ids=input_ids, use_mcdropout=mcdropout)
# QC
if i==0:
eps=1e-5
mpe = lambda x,y: np.mean(np.abs(x-y)/(np.abs(x)+np.abs(y)+eps))
a,b=hs2['hidden_states'],hs1['hidden_states']
assert mpe(a,b)>eps, "the hidden state pairs should be different but are not. Check model.config.use_cache==False, check this model has dropout in it's arch"
# FIXME, move check to loading?
# assert ((hs1['prob_y']+hs1['prob_n'])>0.5).all(), "your chosen binary answers should take up a lot of the prob space, otherwise choose differen't tokens"
else:
hs2 = hs1
for j in range(nn):
# let's add the non torch metadata like label, prompt, lie, etc
k = i*batch_size + j
info = ds_p_subset[k]
yield dict(
hs1=hs1['hidden_states'][j],
scores1=hs1["scores"][j],
hs2=hs2['hidden_states'][j],
scores2=hs2["scores"][j],
true=true_labels[j].item(),
index=index[j],
**info
)
def __getstate__(self):
"""So avoid datasets trying to pickle a model lets set a custom pickle method"""
state = self.__dict__.copy()
state['model_config'] = self.model.config
state['model_name'] = self.model.config
del state['model']
return state
def __setstate__(self):
raise NotImplementedError("You should not be pickling this class, it's too big")
src/datasets/load.py
+3 -1
View File
@@ -6,8 +6,10 @@ def rows_item(row):
transform a row by turning singe dim arrays into items
"""
for k,x in row.items():
if isinstance(x, np.ndarray) and x.ndim==0:
if isinstance(x, np.ndarray) and (x.ndim==0 or (x.ndim==1 and len(x)==1)):
row[k]=x.item()
if isinstance(x, list) and len(x)==1:
row[k]=x[0]
return row
def ds_info2df(ds):