mirror of
https://github.com/wassname/discovering_latent_knowledge.git
synced 2026-09-10 12:00:13 +08:00
99.6%?! n=250
This commit is contained in:
deep1
File diff suppressed because it is too large
Load Diff
+480
-451
File diff suppressed because one or more lines are too long
@@ -0,0 +1,201 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Using matplotlib backend: agg\n",
|
||||
"%pylab is deprecated, use %matplotlib inline and import the required libraries.\n",
|
||||
"Populating the interactive namespace from numpy and matplotlib\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"import numpy as np\n",
|
||||
"import torch.nn.functional as F\n",
|
||||
"%pylab"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 39,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"input tensor([ 1.3567, 0.4950, 1.2330, -0.3437, 0.3804, -1.1041, 1.2604, 0.8007,\n",
|
||||
" 0.7767, -0.9054, 0.5123, 0.1358, 1.4427, -0.0783, 0.3679, -0.6244,\n",
|
||||
" -0.9410, 2.3286, 1.1133, -0.3884, 1.2145, -1.0323, -1.1726, 1.2480,\n",
|
||||
" 0.4702, -0.1345, 0.5357, 0.4737, 0.1690, 0.9409],\n",
|
||||
" requires_grad=True)\n",
|
||||
"target tensor([0., 1., 0., 0., 1., 1., 0., 0., 0., 1., 0., 1., 0., 0., 1., 0., 1., 0.,\n",
|
||||
" 1., 0., 1., 0., 0., 0., 1., 0., 1., 0., 1., 0.])\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"tensor(0.9066, grad_fn=<BinaryCrossEntropyWithLogitsBackward0>)"
|
||||
]
|
||||
},
|
||||
"execution_count": 39,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"\n",
|
||||
"input = torch.randn(30, requires_grad=True)\n",
|
||||
"target = torch.empty(30).random_(2)\n",
|
||||
"print('input', input)\n",
|
||||
"print('target', target)\n",
|
||||
"F.binary_cross_entropy_with_logits(input, target)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 40,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"tensor(0.9066, grad_fn=<BinaryCrossEntropyBackward0>)"
|
||||
]
|
||||
},
|
||||
"execution_count": 40,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"F.binary_cross_entropy(torch.sigmoid(input), target)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 41,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# def dice_loss(true, logits, eps=1e-7):\n",
|
||||
"# \"\"\"Computes the Sørensen–Dice loss.\n",
|
||||
"\n",
|
||||
"# Note that PyTorch optimizers minimize a loss. In this\n",
|
||||
"# case, we would like to maximize the dice loss so we\n",
|
||||
"# return the negated dice loss.\n",
|
||||
"\n",
|
||||
"# Args:\n",
|
||||
"# true: a tensor of shape [B, 1, H, W].\n",
|
||||
"# logits: a tensor of shape [B, C, H, W]. Corresponds to\n",
|
||||
"# the raw output or logits of the model.\n",
|
||||
"# eps: added to the denominator for numerical stability.\n",
|
||||
"\n",
|
||||
"# Returns:\n",
|
||||
"# dice_loss: the Sørensen–Dice loss.\n",
|
||||
"# \"\"\"\n",
|
||||
"# # assert logits.ndim == 2\n",
|
||||
"# num_classes = 1\n",
|
||||
"# true_1_hot = torch.eye(num_classes + 1)[true.long()]\n",
|
||||
"# true_1_hot = true_1_hot.permute(0, 3, 1, 2).float()\n",
|
||||
"# true_1_hot_f = true_1_hot[:, 0:1, :, :]\n",
|
||||
"# true_1_hot_s = true_1_hot[:, 1:2, :, :]\n",
|
||||
"# true_1_hot = torch.cat([true_1_hot_s, true_1_hot_f], dim=1)\n",
|
||||
"# pos_prob = torch.sigmoid(logits)\n",
|
||||
"# neg_prob = 1 - pos_prob\n",
|
||||
"# probas = torch.cat([pos_prob, neg_prob], dim=1)\n",
|
||||
" \n",
|
||||
"# true_1_hot = true_1_hot.type(logits.type())\n",
|
||||
"# dims = (0,) + tuple(range(2, true.ndimension()))\n",
|
||||
"# intersection = torch.sum(probas * true_1_hot, dims)\n",
|
||||
"# cardinality = torch.sum(probas + true_1_hot, dims)\n",
|
||||
"# dice_loss = (2. * intersection / (cardinality + eps)).mean()\n",
|
||||
"# return (1 - dice_loss)\n",
|
||||
"\n",
|
||||
"# dice_loss(input[None, :, None, None], target[None, :, None, None])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 45,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"tensor(0.5394, grad_fn=<RsubBackward1>)"
|
||||
]
|
||||
},
|
||||
"execution_count": 45,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"\n",
|
||||
"def dice_loss(input, target):\n",
|
||||
" smooth = 1.\n",
|
||||
"\n",
|
||||
" iflat = input.view(-1)\n",
|
||||
" tflat = target.view(-1)\n",
|
||||
" intersection = (iflat * tflat).sum()\n",
|
||||
" \n",
|
||||
" return 1 - ((2. * intersection + smooth) /\n",
|
||||
" (iflat.sum() + tflat.sum() + smooth))\n",
|
||||
"\n",
|
||||
"dice_loss(F.sigmoid(input), target)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "dlk3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
+3
-7
@@ -156,10 +156,6 @@ class ExtractHiddenStates:
|
||||
layers = self.get_layer_selection(outputs)
|
||||
head_activation_and_grad = head_activation_and_grad[:, layers]
|
||||
mlp_activation_and_grad = mlp_activation_and_grad[:, layers]
|
||||
# head_activation = head_activation[:, layers]
|
||||
# mlp_activation = mlp_activation[:, layers]
|
||||
# head_activation_grads = head_activation_grads[:, layers]
|
||||
# mlp_activation_grads = mlp_activation_grads[:, layers]
|
||||
hidden_states = hidden_states[:, layers]
|
||||
|
||||
w_grads_mlp_cfc = w_grads_mlp_cfc[:, layers]
|
||||
@@ -179,11 +175,11 @@ class ExtractHiddenStates:
|
||||
|
||||
# head_activation_grads = head_activation_grads,
|
||||
head_activation_and_grad=head_activation_and_grad,
|
||||
mlp_activation_and_grad=mlp_activation_and_grad,
|
||||
# mlp_activation_and_grad=mlp_activation_and_grad,
|
||||
|
||||
# w_grads_mlp=w_grads_mlp,
|
||||
w_grads_mlp_cfc=w_grads_mlp_cfc,
|
||||
w_grads_attn=w_grads_attn,
|
||||
# w_grads_mlp_cfc=w_grads_mlp_cfc,
|
||||
# w_grads_attn=w_grads_attn,
|
||||
)
|
||||
out = {k: detachcpu(v) for k, v in out.items()}
|
||||
if debug:
|
||||
|
||||
Reference in New Issue
Block a user