mirror of
https://github.com/wassname/peft.git
synced 2026-09-24 13:40:15 +08:00
492 KiB
492 KiB
In [ ]:
# ! rm -r unilm
# ! pip install unilmIn [ ]:
# ! wget https://guillaumejaume.github.io/FUNSD/dataset.zip
# ! unzip dataset.zip && mv dataset data && rm -rf dataset.zip __MACOSXIn [1]:
from PIL import Image, ImageDraw, ImageFont
import os
base_path = "/home/sourab/temp/data/dataset"
image = Image.open(os.path.join(base_path, "training_data/images/0000971160.png"))
image = image.convert("RGB")
imageOut [1]:
In [2]:
import json
with open(os.path.join(base_path, 'training_data/annotations/0000971160.json')) as f:
data = json.load(f)
for annotation in data['form']:
print(annotation){'box': [292, 91, 376, 175], 'text': 'R&D', 'label': 'other', 'words': [{'box': [292, 91, 376, 175], 'text': 'R&D'}], 'linking': [], 'id': 0}
{'box': [219, 316, 225, 327], 'text': ':', 'label': 'question', 'words': [{'box': [219, 316, 225, 327], 'text': ':'}], 'linking': [], 'id': 1}
{'box': [95, 355, 169, 370], 'text': 'Suggestion:', 'label': 'question', 'words': [{'box': [95, 355, 169, 370], 'text': 'Suggestion:'}], 'linking': [[2, 16]], 'id': 2}
{'box': [482, 268, 518, 282], 'text': 'Date:', 'label': 'question', 'words': [{'box': [482, 268, 518, 282], 'text': 'Date:'}], 'linking': [[3, 12]], 'id': 3}
{'box': [511, 309, 570, 323], 'text': 'Licensee', 'label': 'answer', 'words': [{'box': [511, 309, 570, 323], 'text': 'Licensee'}], 'linking': [[13, 4]], 'id': 4}
{'box': [211, 651, 217, 662], 'text': '', 'label': 'question', 'words': [{'box': [211, 651, 217, 662], 'text': ''}], 'linking': [], 'id': 5}
{'box': [461, 605, 483, 619], 'text': 'Yes', 'label': 'question', 'words': [{'box': [461, 605, 483, 619], 'text': 'Yes'}], 'linking': [[19, 6]], 'id': 6}
{'box': [545, 603, 563, 617], 'text': 'No', 'label': 'question', 'words': [{'box': [545, 603, 563, 617], 'text': 'No'}], 'linking': [[19, 7]], 'id': 7}
{'box': [525, 904, 641, 926], 'text': '597005708', 'label': 'other', 'words': [{'box': [525, 904, 641, 926], 'text': '597005708'}], 'linking': [], 'id': 8}
{'text': 'R&D QUALITY IMPROVEMENT SUGGESTION/ SOLUTION FORM', 'box': [256, 201, 423, 230], 'linking': [], 'label': 'header', 'words': [{'text': 'R&D', 'box': [257, 203, 279, 214]}, {'text': 'QUALITY', 'box': [285, 203, 334, 216]}, {'text': 'IMPROVEMENT', 'box': [341, 201, 418, 211]}, {'text': 'SUGGESTION/', 'box': [256, 215, 324, 229]}, {'text': '', 'box': [324, 216, 332, 230]}, {'text': 'SOLUTION', 'box': [331, 214, 387, 228]}, {'text': 'FORM', 'box': [395, 215, 423, 228]}], 'id': 9}
{'text': 'Name / Phone Ext. :', 'box': [89, 272, 204, 289], 'linking': [[10, 11]], 'label': 'question', 'words': [{'text': 'Name', 'box': [89, 274, 118, 289]}, {'text': '/', 'box': [117, 274, 127, 288]}, {'text': 'Phone', 'box': [128, 274, 163, 289]}, {'text': 'Ext.', 'box': [169, 272, 196, 287]}, {'text': ':', 'box': [196, 274, 204, 288]}], 'id': 10}
{'text': 'M. Hamann P. Harper, P. Martinez', 'box': [215, 271, 451, 287], 'linking': [[10, 11]], 'label': 'answer', 'words': [{'text': 'M.', 'box': [215, 272, 230, 287]}, {'text': 'Hamann', 'box': [237, 272, 287, 286]}, {'text': 'P.', 'box': [293, 272, 307, 286]}, {'text': 'Harper,', 'box': [314, 274, 363, 285]}, {'text': 'P.', 'box': [370, 272, 384, 285]}, {'text': 'Martinez', 'box': [390, 271, 451, 282]}], 'id': 11}
{'text': '9/ 3/ 92', 'box': [543, 264, 590, 279], 'linking': [[3, 12]], 'label': 'answer', 'words': [{'text': '9/', 'box': [543, 265, 560, 279]}, {'text': '3/', 'box': [560, 264, 575, 279]}, {'text': '92', 'box': [575, 264, 590, 279]}], 'id': 12}
{'text': 'R&D Group:', 'box': [420, 310, 491, 323], 'linking': [[13, 4]], 'label': 'question', 'words': [{'text': 'R&D', 'box': [420, 310, 442, 323]}, {'text': 'Group:', 'box': [448, 310, 491, 323]}], 'id': 13}
{'text': 'J. S. Wigand', 'box': [236, 313, 327, 327], 'linking': [[15, 14]], 'label': 'answer', 'words': [{'text': 'J.', 'box': [236, 313, 251, 327]}, {'text': 'S.', 'box': [256, 313, 273, 326]}, {'text': 'Wigand', 'box': [278, 313, 327, 327]}], 'id': 14}
{'text': 'Supervisor / Manager', 'box': [91, 316, 218, 331], 'linking': [[15, 14]], 'label': 'question', 'words': [{'text': 'Supervisor', 'box': [91, 316, 161, 330]}, {'text': '/', 'box': [163, 318, 169, 331]}, {'text': 'Manager', 'box': [169, 317, 218, 327]}], 'id': 15}
{'text': 'Discontinue coal retention analyses on licensee submitted product samples (Note : Coal Retention testing is not performed by most licensees. Other B&W physical measurements as ends stability and inspection for soft spots in ciparettes are thought to be sufficient measures to assure cigarette physical integrity. The proposed action will increase laboratory productivity . )', 'box': [190, 346, 594, 447], 'linking': [[2, 16]], 'label': 'answer', 'words': [{'text': 'Discontinue', 'box': [190, 355, 268, 366]}, {'text': 'coal', 'box': [274, 353, 303, 366]}, {'text': 'retention', 'box': [309, 352, 375, 365]}, {'text': 'analyses', 'box': [381, 351, 435, 365]}, {'text': 'on', 'box': [443, 352, 458, 363]}, {'text': 'licensee', 'box': [464, 348, 520, 362]}, {'text': 'submitted', 'box': [527, 346, 594, 361]}, {'text': 'product', 'box': [190, 369, 240, 383]}, {'text': 'samples', 'box': [247, 367, 301, 380]}, {'text': '(Note', 'box': [318, 365, 352, 379]}, {'text': ':', 'box': [352, 367, 359, 380]}, {'text': 'Coal', 'box': [373, 366, 402, 376]}, {'text': 'Retention', 'box': [408, 366, 472, 376]}, {'text': 'testing', 'box': [479, 365, 529, 376]}, {'text': 'is', 'box': [536, 363, 549, 374]}, {'text': 'not', 'box': [554, 363, 578, 374]}, {'text': 'performed', 'box': [190, 383, 256, 394]}, {'text': 'by', 'box': [261, 381, 275, 394]}, {'text': 'most', 'box': [282, 383, 311, 393]}, {'text': 'licensees.', 'box': [318, 380, 386, 391]}, {'text': 'Other', 'box': [401, 378, 437, 389]}, {'text': 'B&W', 'box': [443, 378, 465, 389]}, {'text': 'physical', 'box': [471, 377, 528, 391]}, {'text': 'measurements', 'box': [191, 398, 275, 406]}, {'text': 'as', 'box': [282, 397, 297, 405]}, {'text': 'ends', 'box': [304, 394, 332, 405]}, {'text': 'stability', 'box': [339, 394, 402, 405]}, {'text': 'and', 'box': [409, 392, 430, 402]}, {'text': 'inspection', 'box': [437, 392, 508, 403]}, {'text': 'for', 'box': [515, 391, 535, 402]}, {'text': 'soft', 'box': [542, 391, 571, 401]}, {'text': 'spots', 'box': [193, 411, 228, 422]}, {'text': 'in', 'box': [235, 409, 250, 420]}, {'text': 'ciparettes', 'box': [256, 409, 327, 419]}, {'text': 'are', 'box': [332, 408, 352, 418]}, {'text': 'thought', 'box': [360, 406, 410, 419]}, {'text': 'to', 'box': [415, 406, 430, 416]}, {'text': 'be', 'box': [436, 404, 453, 417]}, {'text': 'sufficient', 'box': [458, 405, 529, 415]}, {'text': 'measures', 'box': [535, 405, 592, 415]}, {'text': 'to', 'box': [193, 425, 208, 433]}, {'text': 'assure', 'box': [214, 423, 255, 431]}, {'text': 'cigarette', 'box': [261, 420, 325, 434]}, {'text': 'physical', 'box': [331, 419, 390, 432]}, {'text': 'integrity.', 'box': [395, 418, 463, 431]}, {'text': 'The', 'box': [478, 416, 500, 429]}, {'text': 'proposed', 'box': [506, 418, 566, 431]}, {'text': 'action', 'box': [193, 436, 236, 447]}, {'text': 'will', 'box': [240, 436, 269, 447]}, {'text': 'increase', 'box': [277, 434, 333, 445]}, {'text': 'laboratory', 'box': [339, 433, 410, 446]}, {'text': 'productivity', 'box': [418, 430, 502, 445]}, {'text': '.', 'box': [503, 433, 507, 444]}, {'text': ')', 'box': [508, 430, 514, 444]}], 'id': 16}
{'text': 'Suggested Solutions (s) :', 'box': [95, 486, 250, 504], 'linking': [[17, 18]], 'label': 'question', 'words': [{'text': 'Suggested', 'box': [95, 489, 159, 504]}, {'text': 'Solutions', 'box': [165, 487, 222, 501]}, {'text': '(s)', 'box': [223, 486, 241, 503]}, {'text': ':', 'box': [243, 489, 250, 503]}], 'id': 17}
{'text': 'Delete coal retention from the list of standard analyses performed on licensee submitted product samples. Special requests for coal retention testing could still be submitted on an exception basis.', 'box': [263, 483, 593, 553], 'linking': [[17, 18]], 'label': 'answer', 'words': [{'text': 'Delete', 'box': [263, 486, 306, 500]}, {'text': 'coal', 'box': [313, 486, 341, 499]}, {'text': 'retention', 'box': [348, 486, 412, 497]}, {'text': 'from', 'box': [416, 485, 447, 498]}, {'text': 'the', 'box': [453, 485, 475, 498]}, {'text': 'list', 'box': [480, 483, 508, 496]}, {'text': 'of', 'box': [515, 483, 532, 494]}, {'text': 'standard', 'box': [536, 483, 593, 494]}, {'text': 'analyses', 'box': [264, 501, 320, 514]}, {'text': 'performed', 'box': [324, 501, 392, 512]}, {'text': 'on', 'box': [397, 501, 412, 511]}, {'text': 'licensee', 'box': [419, 499, 475, 512]}, {'text': 'submitted', 'box': [482, 499, 546, 510]}, {'text': 'product', 'box': [264, 517, 314, 528]}, {'text': 'samples.', 'box': [320, 514, 374, 528]}, {'text': 'Special', 'box': [390, 513, 439, 526]}, {'text': 'requests', 'box': [446, 513, 502, 524]}, {'text': 'for', 'box': [508, 511, 530, 522]}, {'text': 'coal', 'box': [538, 510, 566, 523]}, {'text': 'retention', 'box': [263, 529, 330, 540]}, {'text': 'testing', 'box': [335, 527, 387, 540]}, {'text': 'could', 'box': [390, 527, 428, 538]}, {'text': 'still', 'box': [433, 525, 468, 536]}, {'text': 'be', 'box': [473, 525, 488, 535]}, {'text': 'submitted', 'box': [496, 524, 560, 537]}, {'text': 'on', 'box': [566, 524, 584, 537]}, {'text': 'an', 'box': [264, 543, 281, 553]}, {'text': 'exception', 'box': [286, 539, 350, 553]}, {'text': 'basis.', 'box': [355, 541, 397, 551]}], 'id': 18}
{'text': 'Have you contacted your Manager/ Supervisor?', 'box': [96, 608, 398, 624], 'linking': [[19, 6], [19, 7]], 'label': 'header', 'words': [{'text': 'Have', 'box': [96, 612, 127, 623]}, {'text': 'you', 'box': [131, 613, 156, 624]}, {'text': 'contacted', 'box': [161, 612, 225, 623]}, {'text': 'your', 'box': [229, 610, 260, 623]}, {'text': 'Manager/', 'box': [264, 609, 314, 622]}, {'text': '', 'box': [314, 608, 322, 622]}, {'text': 'Supervisor?', 'box': [323, 608, 398, 621]}], 'id': 19}
{'text': 'Manager Comments:', 'box': [98, 651, 211, 665], 'linking': [[20, 21], [20, 22]], 'label': 'question', 'words': [{'text': 'Manager', 'box': [98, 654, 150, 665]}, {'text': 'Comments:', 'box': [154, 651, 211, 664]}], 'id': 20}
{'text': 'Manager, please contact suggester and forward', 'box': [232, 644, 547, 662], 'linking': [[20, 21]], 'label': 'answer', 'words': [{'text': 'Manager,', 'box': [232, 648, 288, 662]}, {'text': 'please', 'box': [296, 649, 338, 662]}, {'text': 'contact', 'box': [344, 648, 394, 662]}, {'text': 'suggester', 'box': [401, 648, 464, 661]}, {'text': 'and', 'box': [469, 647, 491, 658]}, {'text': 'forward', 'box': [497, 644, 547, 657]}], 'id': 21}
{'text': 'comments to the Quality Council.', 'box': [99, 662, 323, 677], 'linking': [[20, 22]], 'label': 'answer', 'words': [{'text': 'comments', 'box': [99, 666, 155, 677]}, {'text': 'to', 'box': [162, 665, 177, 676]}, {'text': 'the', 'box': [183, 665, 205, 675]}, {'text': 'Quality', 'box': [211, 663, 261, 676]}, {'text': 'Council.', 'box': [267, 662, 323, 676]}], 'id': 22}
{'text': 'qip . wp', 'box': [102, 823, 145, 838], 'linking': [], 'label': 'other', 'words': [{'text': 'qip', 'box': [102, 824, 123, 837]}, {'text': '.', 'box': [124, 824, 130, 838]}, {'text': 'wp', 'box': [130, 823, 145, 837]}], 'id': 23}
In [3]:
draw = ImageDraw.Draw(image, "RGBA")
font = ImageFont.load_default()
label2color = {'question':'blue', 'answer':'green', 'header':'orange', 'other':'violet'}
for annotation in data['form']:
label = annotation['label']
general_box = annotation['box']
draw.rectangle(general_box, outline=label2color[label], width=2)
draw.text((general_box[0] + 10, general_box[1] - 10), label, fill=label2color[label], font=font)
words = annotation['words']
for word in words:
box = word['box']
draw.rectangle(box, outline=label2color[label], width=1)
imageOut [3]:
In [4]:
# ! python unilm/layoutlm/examples/seq_labeling/preprocess.py --data_dir data/dataset/training_data/annotations \
# --data_split train \
# --output_dir data \
# --model_name_or_path microsoft/layoutlm-base-uncased \
# --max_len 510
# ! python unilm/layoutlm/examples/seq_labeling/preprocess.py --data_dir data/dataset/testing_data/annotations \
# --data_split test \
# --output_dir data \
# --model_name_or_path microsoft/layoutlm-base-uncased \
# --max_len 510In [5]:
# ! cat data/train.txt | cut -d$'\t' -f 2 | grep -v "^$"| sort | uniq > data/labels.txtIn [6]:
from torch.nn import CrossEntropyLoss
def get_labels(path):
with open(path, "r") as f:
labels = f.read().splitlines()
if "O" not in labels:
labels = ["O"] + labels
return labels
labels = get_labels("data/labels.txt")
num_labels = len(labels)
label_map = {i: label for i, label in enumerate(labels)}
# Use cross entropy ignore index as padding label id so that only real label ids contribute to the loss later
pad_token_label_id = CrossEntropyLoss().ignore_indexIn [7]:
print(labels)['B-ANSWER', 'B-HEADER', 'B-QUESTION', 'E-ANSWER', 'E-HEADER', 'E-QUESTION', 'I-ANSWER', 'I-HEADER', 'I-QUESTION', 'O', 'S-ANSWER', 'S-HEADER', 'S-QUESTION']
In [8]:
import logging
import os
import torch
from torch.utils.data import Dataset
logger = logging.getLogger(__name__)
class FunsdDataset(Dataset):
def __init__(self, args, tokenizer, labels, pad_token_label_id, mode):
if args.local_rank not in [-1, 0] and mode == "train":
torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache
# Load data features from cache or dataset file
cached_features_file = os.path.join(
args.data_dir,
"cached_{}_{}_{}".format(
mode,
list(filter(None, args.model_name_or_path.split("/"))).pop(),
str(args.max_seq_length),
),
)
if os.path.exists(cached_features_file) and not args.overwrite_cache:
logger.info("Loading features from cached file %s", cached_features_file)
features = torch.load(cached_features_file)
else:
logger.info("Creating features from dataset file at %s", args.data_dir)
examples = read_examples_from_file(args.data_dir, mode)
features = convert_examples_to_features(
examples,
labels,
args.max_seq_length,
tokenizer,
cls_token_at_end=bool(args.model_type in ["xlnet"]),
# xlnet has a cls token at the end
cls_token=tokenizer.cls_token,
cls_token_segment_id=2 if args.model_type in ["xlnet"] else 0,
sep_token=tokenizer.sep_token,
sep_token_extra=bool(args.model_type in ["roberta"]),
# roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805
pad_on_left=bool(args.model_type in ["xlnet"]),
# pad on the left for xlnet
pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
pad_token_segment_id=4 if args.model_type in ["xlnet"] else 0,
pad_token_label_id=pad_token_label_id,
)
#if args.local_rank in [-1, 0]:
#logger.info("Saving features into cached file %s", cached_features_file)
#torch.save(features, cached_features_file)
if args.local_rank == 0 and mode == "train":
torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache
self.features = features
# Convert to Tensors and build dataset
self.all_input_ids = torch.tensor(
[f.input_ids for f in features], dtype=torch.long
)
self.all_input_mask = torch.tensor(
[f.input_mask for f in features], dtype=torch.long
)
self.all_segment_ids = torch.tensor(
[f.segment_ids for f in features], dtype=torch.long
)
self.all_label_ids = torch.tensor(
[f.label_ids for f in features], dtype=torch.long
)
self.all_bboxes = torch.tensor([f.boxes for f in features], dtype=torch.long)
def __len__(self):
return len(self.features)
def __getitem__(self, index):
return (
self.all_input_ids[index],
self.all_input_mask[index],
self.all_segment_ids[index],
self.all_label_ids[index],
self.all_bboxes[index],
)
class InputExample(object):
"""A single training/test example for token classification."""
def __init__(self, guid, words, labels, boxes, actual_bboxes, file_name, page_size):
"""Constructs a InputExample.
Args:
guid: Unique id for the example.
words: list. The words of the sequence.
labels: (Optional) list. The labels for each word of the sequence. This should be
specified for train and dev examples, but not for test examples.
"""
self.guid = guid
self.words = words
self.labels = labels
self.boxes = boxes
self.actual_bboxes = actual_bboxes
self.file_name = file_name
self.page_size = page_size
class InputFeatures(object):
"""A single set of features of data."""
def __init__(
self,
input_ids,
input_mask,
segment_ids,
label_ids,
boxes,
actual_bboxes,
file_name,
page_size,
):
assert (
0 <= all(boxes) <= 1000
), "Error with input bbox ({}): the coordinate value is not between 0 and 1000".format(
boxes
)
self.input_ids = input_ids
self.input_mask = input_mask
self.segment_ids = segment_ids
self.label_ids = label_ids
self.boxes = boxes
self.actual_bboxes = actual_bboxes
self.file_name = file_name
self.page_size = page_size
def read_examples_from_file(data_dir, mode):
file_path = os.path.join(data_dir, "{}.txt".format(mode))
box_file_path = os.path.join(data_dir, "{}_box.txt".format(mode))
image_file_path = os.path.join(data_dir, "{}_image.txt".format(mode))
guid_index = 1
examples = []
with open(file_path, encoding="utf-8") as f, open(
box_file_path, encoding="utf-8"
) as fb, open(image_file_path, encoding="utf-8") as fi:
words = []
boxes = []
actual_bboxes = []
file_name = None
page_size = None
labels = []
for line, bline, iline in zip(f, fb, fi):
if line.startswith("-DOCSTART-") or line == "" or line == "\n":
if words:
examples.append(
InputExample(
guid="{}-{}".format(mode, guid_index),
words=words,
labels=labels,
boxes=boxes,
actual_bboxes=actual_bboxes,
file_name=file_name,
page_size=page_size,
)
)
guid_index += 1
words = []
boxes = []
actual_bboxes = []
file_name = None
page_size = None
labels = []
else:
splits = line.split("\t")
bsplits = bline.split("\t")
isplits = iline.split("\t")
assert len(splits) == 2
assert len(bsplits) == 2
assert len(isplits) == 4
assert splits[0] == bsplits[0]
words.append(splits[0])
if len(splits) > 1:
labels.append(splits[-1].replace("\n", ""))
box = bsplits[-1].replace("\n", "")
box = [int(b) for b in box.split()]
boxes.append(box)
actual_bbox = [int(b) for b in isplits[1].split()]
actual_bboxes.append(actual_bbox)
page_size = [int(i) for i in isplits[2].split()]
file_name = isplits[3].strip()
else:
# Examples could have no label for mode = "test"
labels.append("O")
if words:
examples.append(
InputExample(
guid="%s-%d".format(mode, guid_index),
words=words,
labels=labels,
boxes=boxes,
actual_bboxes=actual_bboxes,
file_name=file_name,
page_size=page_size,
)
)
return examples
def convert_examples_to_features(
examples,
label_list,
max_seq_length,
tokenizer,
cls_token_at_end=False,
cls_token="[CLS]",
cls_token_segment_id=1,
sep_token="[SEP]",
sep_token_extra=False,
pad_on_left=False,
pad_token=0,
cls_token_box=[0, 0, 0, 0],
sep_token_box=[1000, 1000, 1000, 1000],
pad_token_box=[0, 0, 0, 0],
pad_token_segment_id=0,
pad_token_label_id=-1,
sequence_a_segment_id=0,
mask_padding_with_zero=True,
):
""" Loads a data file into a list of `InputBatch`s
`cls_token_at_end` define the location of the CLS token:
- False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]
- True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]
`cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet)
"""
label_map = {label: i for i, label in enumerate(label_list)}
features = []
for (ex_index, example) in enumerate(examples):
file_name = example.file_name
page_size = example.page_size
width, height = page_size
if ex_index % 10000 == 0:
logger.info("Writing example %d of %d", ex_index, len(examples))
tokens = []
token_boxes = []
actual_bboxes = []
label_ids = []
for word, label, box, actual_bbox in zip(
example.words, example.labels, example.boxes, example.actual_bboxes
):
word_tokens = tokenizer.tokenize(word)
tokens.extend(word_tokens)
token_boxes.extend([box] * len(word_tokens))
actual_bboxes.extend([actual_bbox] * len(word_tokens))
# Use the real label id for the first token of the word, and padding ids for the remaining tokens
label_ids.extend(
[label_map[label]] + [pad_token_label_id] * (len(word_tokens) - 1)
)
# Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
special_tokens_count = 3 if sep_token_extra else 2
if len(tokens) > max_seq_length - special_tokens_count:
tokens = tokens[: (max_seq_length - special_tokens_count)]
token_boxes = token_boxes[: (max_seq_length - special_tokens_count)]
actual_bboxes = actual_bboxes[: (max_seq_length - special_tokens_count)]
label_ids = label_ids[: (max_seq_length - special_tokens_count)]
# The convention in BERT is:
# (a) For sequence pairs:
# tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
# type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1
# (b) For single sequences:
# tokens: [CLS] the dog is hairy . [SEP]
# type_ids: 0 0 0 0 0 0 0
#
# Where "type_ids" are used to indicate whether this is the first
# sequence or the second sequence. The embedding vectors for `type=0` and
# `type=1` were learned during pre-training and are added to the wordpiece
# embedding vector (and position vector). This is not *strictly* necessary
# since the [SEP] token unambiguously separates the sequences, but it makes
# it easier for the model to learn the concept of sequences.
#
# For classification tasks, the first vector (corresponding to [CLS]) is
# used as as the "sentence vector". Note that this only makes sense because
# the entire model is fine-tuned.
tokens += [sep_token]
token_boxes += [sep_token_box]
actual_bboxes += [[0, 0, width, height]]
label_ids += [pad_token_label_id]
if sep_token_extra:
# roberta uses an extra separator b/w pairs of sentences
tokens += [sep_token]
token_boxes += [sep_token_box]
actual_bboxes += [[0, 0, width, height]]
label_ids += [pad_token_label_id]
segment_ids = [sequence_a_segment_id] * len(tokens)
if cls_token_at_end:
tokens += [cls_token]
token_boxes += [cls_token_box]
actual_bboxes += [[0, 0, width, height]]
label_ids += [pad_token_label_id]
segment_ids += [cls_token_segment_id]
else:
tokens = [cls_token] + tokens
token_boxes = [cls_token_box] + token_boxes
actual_bboxes = [[0, 0, width, height]] + actual_bboxes
label_ids = [pad_token_label_id] + label_ids
segment_ids = [cls_token_segment_id] + segment_ids
input_ids = tokenizer.convert_tokens_to_ids(tokens)
# The mask has 1 for real tokens and 0 for padding tokens. Only real
# tokens are attended to.
input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
# Zero-pad up to the sequence length.
padding_length = max_seq_length - len(input_ids)
if pad_on_left:
input_ids = ([pad_token] * padding_length) + input_ids
input_mask = (
[0 if mask_padding_with_zero else 1] * padding_length
) + input_mask
segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids
label_ids = ([pad_token_label_id] * padding_length) + label_ids
token_boxes = ([pad_token_box] * padding_length) + token_boxes
else:
input_ids += [pad_token] * padding_length
input_mask += [0 if mask_padding_with_zero else 1] * padding_length
segment_ids += [pad_token_segment_id] * padding_length
label_ids += [pad_token_label_id] * padding_length
token_boxes += [pad_token_box] * padding_length
assert len(input_ids) == max_seq_length
assert len(input_mask) == max_seq_length
assert len(segment_ids) == max_seq_length
assert len(label_ids) == max_seq_length
assert len(token_boxes) == max_seq_length
if ex_index < 5:
logger.info("*** Example ***")
logger.info("guid: %s", example.guid)
logger.info("tokens: %s", " ".join([str(x) for x in tokens]))
logger.info("input_ids: %s", " ".join([str(x) for x in input_ids]))
logger.info("input_mask: %s", " ".join([str(x) for x in input_mask]))
logger.info("segment_ids: %s", " ".join([str(x) for x in segment_ids]))
logger.info("label_ids: %s", " ".join([str(x) for x in label_ids]))
logger.info("boxes: %s", " ".join([str(x) for x in token_boxes]))
logger.info("actual_bboxes: %s", " ".join([str(x) for x in actual_bboxes]))
features.append(
InputFeatures(
input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
label_ids=label_ids,
boxes=token_boxes,
actual_bboxes=actual_bboxes,
file_name=file_name,
page_size=page_size,
)
)
return features
In [9]:
from transformers import LayoutLMTokenizer
#from .unilm.layoutlm.data.funsd import FunsdDataset, InputFeatures
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
batch_size = 16
args = {'local_rank': -1,
'overwrite_cache': True,
'data_dir': '/home/sourab/temp/data/',
'model_name_or_path':'microsoft/layoutlm-base-uncased',
'max_seq_length': 512,
'model_type': 'layoutlm',
}
# class to turn the keys of a dict into attributes (thanks Stackoverflow)
class AttrDict(dict):
def __init__(self, *args, **kwargs):
super(AttrDict, self).__init__(*args, **kwargs)
self.__dict__ = self
args = AttrDict(args)
tokenizer = LayoutLMTokenizer.from_pretrained("microsoft/layoutlm-base-uncased")
# the LayoutLM authors already defined a specific FunsdDataset, so we are going to use this here
train_dataset = FunsdDataset(args, tokenizer, labels, pad_token_label_id, mode="train")
train_sampler = RandomSampler(train_dataset)
train_dataloader = DataLoader(train_dataset,
sampler=train_sampler,
batch_size=batch_size)
eval_dataset = FunsdDataset(args, tokenizer, labels, pad_token_label_id, mode="test")
eval_sampler = SequentialSampler(eval_dataset)
eval_dataloader = DataLoader(eval_dataset,
sampler=eval_sampler,
batch_size=batch_size)In [10]:
len(train_dataloader)Out [10]:
10
In [11]:
len(eval_dataloader)Out [11]:
4
In [12]:
batch = next(iter(train_dataloader))
input_ids = batch[0][0]
tokenizer.decode(input_ids)Out [12]:
'[CLS] project objective : date : btf - to : files confidential epb : cc : odl project initiation form september 16, 1980 project code : project name : project leader : work requested by : e. p. barbee r. s. sprinkle, iii to develop cigarette cigarette to utilize a filter tip with longitudinal grooves from the mouth end to the tobacco end in con - junction with perforated tipping paper. other personnel assigned : j. e. mann, jr. d. e. cawthon approved by : r. s. sprinkle, iii bmc / rmi / epb jem / dec prc / drb [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]'
In [13]:
from pet import get_pet_config, LoRAModel, get_pet_model, LoRAConfig, TaskType
pet_config = LoRAConfig(
task_type=TaskType.TOKEN_CLS,
inference_mode=False,
r=16,
lora_alpha=16,
lora_dropout=0.1,
bias="all"
)
pet_configOut [13]:
LoRAConfig(pet_type='LORA', task_type='TOKEN_CLS', inference_mode=False, r=16, target_modules=None, lora_alpha=16, lora_dropout=0.1, merge_weights=False, fan_in_fan_out=False, enable_lora=None, bias='all')
In [14]:
from transformers import LayoutLMForTokenClassification
import torch
from transformers import set_seed
seed = 100
set_seed(seed)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = LayoutLMForTokenClassification.from_pretrained("microsoft/layoutlm-base-uncased", num_labels=num_labels)
model = get_pet_model(model, pet_config)
model.to(device)Out [14]:
Some weights of the model checkpoint at microsoft/layoutlm-base-uncased were not used when initializing LayoutLMForTokenClassification: ['cls.predictions.transform.dense.bias', 'cls.predictions.decoder.bias', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight'] - This IS expected if you are initializing LayoutLMForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model). - This IS NOT expected if you are initializing LayoutLMForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model). Some weights of LayoutLMForTokenClassification were not initialized from the model checkpoint at microsoft/layoutlm-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias'] You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
PETModelForTokenClassification(
(base_model): LoRAModel(
(model): LayoutLMForTokenClassification(
(layoutlm): LayoutLMModel(
(embeddings): LayoutLMEmbeddings(
(word_embeddings): Embedding(30522, 768, padding_idx=0)
(position_embeddings): Embedding(512, 768)
(x_position_embeddings): Embedding(1024, 768)
(y_position_embeddings): Embedding(1024, 768)
(h_position_embeddings): Embedding(1024, 768)
(w_position_embeddings): Embedding(1024, 768)
(token_type_embeddings): Embedding(2, 768)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
(encoder): LayoutLMEncoder(
(layer): ModuleList(
(0): LayoutLMLayer(
(attention): LayoutLMAttention(
(self): LayoutLMSelfAttention(
(query): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(output): LayoutLMSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(intermediate): LayoutLMIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
(intermediate_act_fn): GELUActivation()
)
(output): LayoutLMOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(1): LayoutLMLayer(
(attention): LayoutLMAttention(
(self): LayoutLMSelfAttention(
(query): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(output): LayoutLMSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(intermediate): LayoutLMIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
(intermediate_act_fn): GELUActivation()
)
(output): LayoutLMOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(2): LayoutLMLayer(
(attention): LayoutLMAttention(
(self): LayoutLMSelfAttention(
(query): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(output): LayoutLMSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(intermediate): LayoutLMIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
(intermediate_act_fn): GELUActivation()
)
(output): LayoutLMOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(3): LayoutLMLayer(
(attention): LayoutLMAttention(
(self): LayoutLMSelfAttention(
(query): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(output): LayoutLMSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(intermediate): LayoutLMIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
(intermediate_act_fn): GELUActivation()
)
(output): LayoutLMOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(4): LayoutLMLayer(
(attention): LayoutLMAttention(
(self): LayoutLMSelfAttention(
(query): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(output): LayoutLMSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(intermediate): LayoutLMIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
(intermediate_act_fn): GELUActivation()
)
(output): LayoutLMOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(5): LayoutLMLayer(
(attention): LayoutLMAttention(
(self): LayoutLMSelfAttention(
(query): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(output): LayoutLMSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(intermediate): LayoutLMIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
(intermediate_act_fn): GELUActivation()
)
(output): LayoutLMOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(6): LayoutLMLayer(
(attention): LayoutLMAttention(
(self): LayoutLMSelfAttention(
(query): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(output): LayoutLMSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(intermediate): LayoutLMIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
(intermediate_act_fn): GELUActivation()
)
(output): LayoutLMOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(7): LayoutLMLayer(
(attention): LayoutLMAttention(
(self): LayoutLMSelfAttention(
(query): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(output): LayoutLMSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(intermediate): LayoutLMIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
(intermediate_act_fn): GELUActivation()
)
(output): LayoutLMOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(8): LayoutLMLayer(
(attention): LayoutLMAttention(
(self): LayoutLMSelfAttention(
(query): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(output): LayoutLMSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(intermediate): LayoutLMIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
(intermediate_act_fn): GELUActivation()
)
(output): LayoutLMOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(9): LayoutLMLayer(
(attention): LayoutLMAttention(
(self): LayoutLMSelfAttention(
(query): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(output): LayoutLMSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(intermediate): LayoutLMIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
(intermediate_act_fn): GELUActivation()
)
(output): LayoutLMOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(10): LayoutLMLayer(
(attention): LayoutLMAttention(
(self): LayoutLMSelfAttention(
(query): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(output): LayoutLMSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(intermediate): LayoutLMIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
(intermediate_act_fn): GELUActivation()
)
(output): LayoutLMOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(11): LayoutLMLayer(
(attention): LayoutLMAttention(
(self): LayoutLMSelfAttention(
(query): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(
in_features=768, out_features=768, bias=True
(lora_dropout): Dropout(p=0.1, inplace=False)
(lora_A): Linear(in_features=768, out_features=16, bias=False)
(lora_B): Linear(in_features=16, out_features=768, bias=False)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(output): LayoutLMSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
(intermediate): LayoutLMIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
(intermediate_act_fn): GELUActivation()
)
(output): LayoutLMOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
)
)
(pooler): LayoutLMPooler(
(dense): Linear(in_features=768, out_features=768, bias=True)
(activation): Tanh()
)
)
(dropout): Dropout(p=0.1, inplace=False)
(classifier): Linear(in_features=768, out_features=13, bias=True)
)
)
)In [15]:
print(model.model.layoutlm.encoder.layer[0].attention.self.query.weight)
print(model.model.layoutlm.encoder.layer[0].attention.self.query.lora_A.weight)
print(model.model.classifier.weight)
Parameter containing:
tensor([[-0.0224, 0.0299, -0.0252, ..., -0.0109, 0.0444, 0.0079],
[-0.0067, 0.0337, -0.0272, ..., -0.0594, 0.1091, 0.0007],
[ 0.0390, 0.0659, 0.0154, ..., -0.0101, 0.0157, -0.0150],
...,
[ 0.0020, 0.0183, 0.0587, ..., 0.0487, -0.0090, -0.0306],
[-0.0447, 0.0733, 0.0809, ..., -0.0755, 0.0394, 0.0626],
[ 0.0111, -0.0696, -0.0267, ..., -0.0041, -0.0576, -0.0373]],
device='cuda:0')
Parameter containing:
tensor([[-0.0253, 0.0085, 0.0225, ..., 0.0184, -0.0036, -0.0280],
[ 0.0029, -0.0113, -0.0316, ..., 0.0294, -0.0333, -0.0033],
[ 0.0151, 0.0121, 0.0291, ..., -0.0328, -0.0295, -0.0161],
...,
[-0.0248, -0.0324, -0.0055, ..., 0.0129, -0.0264, -0.0004],
[ 0.0191, 0.0314, 0.0033, ..., -0.0264, 0.0292, 0.0079],
[ 0.0197, 0.0346, -0.0040, ..., 0.0037, 0.0151, -0.0032]],
device='cuda:0', requires_grad=True)
Parameter containing:
tensor([[ 3.3286e-03, 2.5395e-04, 1.2631e-02, ..., -2.2320e-02,
3.2886e-02, 1.8957e-02],
[ 3.3295e-02, -2.8352e-02, -1.4806e-02, ..., -6.0922e-04,
-9.3333e-05, -1.5491e-02],
[ 1.4130e-02, -4.7829e-03, -1.1069e-02, ..., 2.9516e-02,
-4.1938e-03, -9.6518e-04],
...,
[-7.7986e-04, -1.9359e-02, 1.9118e-02, ..., -1.6913e-02,
-3.6974e-03, -1.3698e-02],
[ 1.1183e-02, -1.6715e-02, 7.0737e-03, ..., -2.1076e-02,
-1.6323e-02, -4.1332e-03],
[-2.9230e-02, 5.7393e-05, 1.9348e-02, ..., -1.4401e-02,
2.9383e-03, 9.1288e-03]], device='cuda:0', requires_grad=True)
In [16]:
from transformers import AdamW, get_linear_schedule_with_warmup
from tqdm import tqdm
num_train_epochs = 100
optimizer = torch.optim.AdamW(model.parameters(), lr=3e-3)
lr_scheduler = get_linear_schedule_with_warmup(
optimizer=optimizer,
num_warmup_steps=0.06*(len(train_dataloader) * num_train_epochs),
num_training_steps=(len(train_dataloader) * num_train_epochs),
)
global_step = 0
t_total = len(train_dataloader) * num_train_epochs # total number of training steps
#put the model in training mode
model.train()
for epoch in range(num_train_epochs):
for batch in tqdm(train_dataloader, desc="Training"):
input_ids = batch[0].to(device)
bbox = batch[4].to(device)
attention_mask = batch[1].to(device)
token_type_ids = batch[2].to(device)
labels = batch[3].to(device)
# forward pass
outputs = model(input_ids=input_ids, bbox=bbox, attention_mask=attention_mask, token_type_ids=token_type_ids,
labels=labels)
loss = outputs.loss
# print loss every 100 steps
if global_step % 10 == 0:
print(f"Loss after {global_step} steps: {loss.item()}")
# backward pass to get the gradients
loss.backward()
#print("Gradients on classification head:")
#print(model.classifier.weight.grad[6,:].sum())
# update
optimizer.step()
lr_scheduler.step()
optimizer.zero_grad()
global_step += 1Training: 10%|█████▍ | 1/10 [00:01<00:09, 1.11s/it]
Loss after 0 steps: 2.6514651775360107
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:03<00:00, 2.87it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 10 steps: 2.384181022644043
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.75it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 20 steps: 2.183215856552124
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 30 steps: 1.7022178173065186
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 40 steps: 1.1768041849136353
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.54it/s]
Loss after 50 steps: 0.9286957383155823
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.53it/s]
Loss after 60 steps: 0.7891858816146851
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 70 steps: 0.6700936555862427
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 80 steps: 0.5903935432434082
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 90 steps: 0.46567460894584656
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 100 steps: 0.43472540378570557
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 110 steps: 0.4344632029533386
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 120 steps: 0.29618480801582336
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 130 steps: 0.3432932496070862
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 140 steps: 0.26484620571136475
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 150 steps: 0.3635620176792145
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 160 steps: 0.28532707691192627
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 170 steps: 0.2699059247970581
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 180 steps: 0.25385379791259766
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 190 steps: 0.23770077526569366
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 200 steps: 0.18621960282325745
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 210 steps: 0.22020076215267181
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 220 steps: 0.16716673970222473
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 230 steps: 0.12251073122024536
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 240 steps: 0.12013913691043854
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 250 steps: 0.09276405721902847
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 260 steps: 0.11919382959604263
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 270 steps: 0.12163368612527847
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 280 steps: 0.09964565187692642
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 290 steps: 0.2966587543487549
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.53it/s]
Loss after 300 steps: 0.1136971265077591
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 310 steps: 0.15149033069610596
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 320 steps: 0.11433174461126328
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 330 steps: 0.08340273052453995
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 340 steps: 0.05601643770933151
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 350 steps: 0.06016877666115761
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 360 steps: 0.06651942431926727
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 370 steps: 0.38789188861846924
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 380 steps: 0.16247011721134186
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 390 steps: 0.10822084546089172
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 400 steps: 0.10201780498027802
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 410 steps: 0.09159471094608307
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 420 steps: 0.052561257034540176
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 430 steps: 0.07446825504302979
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 440 steps: 0.055142153054475784
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 450 steps: 0.04064352810382843
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 460 steps: 0.047722212970256805
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 470 steps: 0.03962016478180885
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 480 steps: 0.05285123735666275
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 490 steps: 0.04547346383333206
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 500 steps: 0.035933323204517365
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 510 steps: 0.01684512570500374
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 520 steps: 0.018589401617646217
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.48it/s]
Loss after 530 steps: 0.034401122480630875
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 540 steps: 0.029914753511548042
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 550 steps: 0.019160285592079163
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 560 steps: 0.030683889985084534
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 570 steps: 0.0281855259090662
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 580 steps: 0.019448189064860344
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 590 steps: 0.01689102128148079
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 600 steps: 0.012563714757561684
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 610 steps: 0.017075713723897934
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]
Loss after 620 steps: 0.011337793432176113
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 630 steps: 0.01142528560012579
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.71it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 640 steps: 0.012161443941295147
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 650 steps: 0.01025047805160284
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 660 steps: 0.008623240515589714
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.48it/s]
Loss after 670 steps: 0.020196430385112762
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 680 steps: 0.013619341887533665
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 690 steps: 0.014648614451289177
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 700 steps: 0.012998558580875397
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 710 steps: 0.021816464141011238
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.71it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 720 steps: 0.01582351326942444
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 730 steps: 0.014046266674995422
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 740 steps: 0.007438092492520809
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.47it/s]
Loss after 750 steps: 0.01699301227927208
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 760 steps: 0.013121521100401878
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 770 steps: 0.01430545188486576
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 780 steps: 0.016542932018637657
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 790 steps: 0.008827055804431438
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 800 steps: 0.014699053950607777
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 810 steps: 0.01083141379058361
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 820 steps: 0.011546650901436806
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 830 steps: 0.007675974629819393
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.71it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 840 steps: 0.011567591689527035
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 850 steps: 0.006588640157133341
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 860 steps: 0.012320021167397499
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 870 steps: 0.021453894674777985
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 880 steps: 0.010092893615365028
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 890 steps: 0.009351171553134918
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 900 steps: 0.004404681734740734
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.48it/s]
Loss after 910 steps: 0.002058168640360236
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.71it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 920 steps: 0.0021678945049643517
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 930 steps: 0.006458617746829987
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 940 steps: 0.007828252390027046
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 950 steps: 0.008966044522821903
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.71it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 960 steps: 0.014970815740525723
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.71it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]
Loss after 970 steps: 0.0066721560433506966
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]
Loss after 980 steps: 0.004333506338298321
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s] Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]
Loss after 990 steps: 0.006984252016991377
Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]
In [17]:
import numpy as np
from seqeval.metrics import (
classification_report,
f1_score,
precision_score,
recall_score,
)
eval_loss = 0.0
nb_eval_steps = 0
preds = None
out_label_ids = None
# put model in evaluation mode
model.eval()
for batch in tqdm(eval_dataloader, desc="Evaluating"):
with torch.no_grad():
input_ids = batch[0].to(device)
bbox = batch[4].to(device)
attention_mask = batch[1].to(device)
token_type_ids = batch[2].to(device)
labels = batch[3].to(device)
# forward pass
outputs = model(input_ids=input_ids, bbox=bbox, attention_mask=attention_mask, token_type_ids=token_type_ids,
labels=labels)
# get the loss and logits
tmp_eval_loss = outputs.loss
logits = outputs.logits
eval_loss += tmp_eval_loss.item()
nb_eval_steps += 1
# compute the predictions
if preds is None:
preds = logits.detach().cpu().numpy()
out_label_ids = labels.detach().cpu().numpy()
else:
preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
out_label_ids = np.append(
out_label_ids, labels.detach().cpu().numpy(), axis=0
)
# compute average evaluation loss
eval_loss = eval_loss / nb_eval_steps
preds = np.argmax(preds, axis=2)
out_label_list = [[] for _ in range(out_label_ids.shape[0])]
preds_list = [[] for _ in range(out_label_ids.shape[0])]
for i in range(out_label_ids.shape[0]):
for j in range(out_label_ids.shape[1]):
if out_label_ids[i, j] != pad_token_label_id:
out_label_list[i].append(label_map[out_label_ids[i][j]])
preds_list[i].append(label_map[preds[i][j]])
results = {
"loss": eval_loss,
"precision": precision_score(out_label_list, preds_list),
"recall": recall_score(out_label_list, preds_list),
"f1": f1_score(out_label_list, preds_list),
}
print(results)Evaluating: 100%|█████████████████████████████████████████████████████| 4/4 [00:00<00:00, 9.73it/s]
{'loss': 1.7456395626068115, 'precision': 0.7407071622846781, 'recall': 0.817408704352176, 'f1': 0.7771700356718193}
In [18]:
model.print_trainable_parameters()trainable params: 702733 || all params: 113227789 || trainable%: 0.6206365117665593
In [19]:
from pet import get_pet_model_state_dict
to_return = get_pet_model_state_dict(model)
In [20]:
torch.save(to_return, "layoutlm_funsd.pt")In [21]:
!du -h "layoutlm_funsd.pt"2,8M layoutlm_funsd.pt
In [ ]: