discovering_latent_knowledge/notebooks/101_scratch.ipynb

In [7]:

import torch
import numpy as np
import torch.nn.functional as F
%pylab

Using matplotlib backend: agg
%pylab is deprecated, use %matplotlib inline and import the required libraries.
Populating the interactive namespace from numpy and matplotlib

In [39]:

input = torch.randn(30, requires_grad=True)
target = torch.empty(30).random_(2)
print('input', input)
print('target', target)
F.binary_cross_entropy_with_logits(input, target)

Out [39]:

input tensor([ 1.3567,  0.4950,  1.2330, -0.3437,  0.3804, -1.1041,  1.2604,  0.8007,
         0.7767, -0.9054,  0.5123,  0.1358,  1.4427, -0.0783,  0.3679, -0.6244,
        -0.9410,  2.3286,  1.1133, -0.3884,  1.2145, -1.0323, -1.1726,  1.2480,
         0.4702, -0.1345,  0.5357,  0.4737,  0.1690,  0.9409],
       requires_grad=True)
target tensor([0., 1., 0., 0., 1., 1., 0., 0., 0., 1., 0., 1., 0., 0., 1., 0., 1., 0.,
        1., 0., 1., 0., 0., 0., 1., 0., 1., 0., 1., 0.])

tensor(0.9066, grad_fn=<BinaryCrossEntropyWithLogitsBackward0>)

In [40]:

F.binary_cross_entropy(torch.sigmoid(input), target)

Out [40]:

tensor(0.9066, grad_fn=<BinaryCrossEntropyBackward0>)

In [41]:

# def dice_loss(true, logits, eps=1e-7):
#     """Computes the Sørensen–Dice loss.

#     Note that PyTorch optimizers minimize a loss. In this
#     case, we would like to maximize the dice loss so we
#     return the negated dice loss.

#     Args:
#         true: a tensor of shape [B, 1, H, W].
#         logits: a tensor of shape [B, C, H, W]. Corresponds to
#             the raw output or logits of the model.
#         eps: added to the denominator for numerical stability.

#     Returns:
#         dice_loss: the Sørensen–Dice loss.
#     """
#     # assert logits.ndim == 2
#     num_classes = 1
#     true_1_hot = torch.eye(num_classes + 1)[true.long()]
#     true_1_hot = true_1_hot.permute(0, 3, 1, 2).float()
#     true_1_hot_f = true_1_hot[:, 0:1, :, :]
#     true_1_hot_s = true_1_hot[:, 1:2, :, :]
#     true_1_hot = torch.cat([true_1_hot_s, true_1_hot_f], dim=1)
#     pos_prob = torch.sigmoid(logits)
#     neg_prob = 1 - pos_prob
#     probas = torch.cat([pos_prob, neg_prob], dim=1)
    
#     true_1_hot = true_1_hot.type(logits.type())
#     dims = (0,) + tuple(range(2, true.ndimension()))
#     intersection = torch.sum(probas * true_1_hot, dims)
#     cardinality = torch.sum(probas + true_1_hot, dims)
#     dice_loss = (2. * intersection / (cardinality + eps)).mean()
#     return (1 - dice_loss)

# dice_loss(input[None, :, None, None], target[None, :, None, None])

In [45]:

def dice_loss(input, target):
    smooth = 1.

    iflat = input.view(-1)
    tflat = target.view(-1)
    intersection = (iflat * tflat).sum()
    
    return 1 - ((2. * intersection + smooth) /
              (iflat.sum() + tflat.sum() + smooth))

dice_loss(F.sigmoid(input), target)

Out [45]:

tensor(0.5394, grad_fn=<RsubBackward1>)

promtps

In [39]:

from src.prompts.prompt_loading import DatasetTemplates, _convert_to_prompts, Random, default_sys_instructions
ds_name = 'imdb'
example = dict(label=0, text= 'text', content="content", title='title', response="Negative")
ds_name = 'amazon_polarity'
example = dict(label=0, text= 'text', content="content", title='title', response="Negative")
prompter = DatasetTemplates(ds_name)
templates = list(prompter.templates.values())
template = templates[0]
template.jinja

Out [39]:

'Title: {{title}}\nReview: {{content}}\nIs the review positive or negative? |||\n{{answer_choices[label]}}'

In [40]:

template.apply(example)

Out [40]:

['Title: title\nReview: content\nIs the review positive or negative?',
 '\nNegative']

In [35]:

sys_instructions = 'say a lie'
rng = Random(42)
prompts = _convert_to_prompts(
    example,
    binarize=True,
    label_column='label',
    label_choices=['No', 'Yes'],  # type: ignore[arg-type]
    prompter=prompter,
    rng=rng,
    # sys_instructions=default_sys_instructions,
    # fewshot_iter=fewshot_iter,
    prompt_format='llama',
)
prompts[0]

Out [35]:

{'answer': 'Negative',
 'question': 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n### Instruction\nTitle: title\nReview: content\nIs the review positive or negative?\n\n### Response:\n',
 'answer_choices': ['Negative', 'Positive'],
 'template_name': 'Is_this_review',
 'label_true': 0,
 'label_instructed': 0,
 'instructed_to_lie': False,
 'sys_instr_name': 'truth'}

In [19]:

# %debug

In [ ]: