mirror of
https://github.com/wassname/multifit.git
synced 2026-09-09 11:27:26 +08:00
Remove previous multi field patch
This commit is contained in:
@@ -148,16 +148,6 @@ class CLSHyperParams(LMHyperParams):
|
||||
**kwargs)
|
||||
return self.databunches(bs, **data)
|
||||
|
||||
def merge_cols(self, df):
|
||||
if len(df.columns) <= 2:
|
||||
return df
|
||||
ndf = df[[0,1]].copy()
|
||||
for i in range(2, len(df.columns)):
|
||||
ndf[1] += ("\n" + FLD + "\n") + df[i].fillna(" ")
|
||||
|
||||
assert ndf[1].isna().sum().sum() == 0, f"You have NaN values in column(s) of your dataframe, please fix it."
|
||||
return ndf
|
||||
|
||||
def load_data(self, lang='', **kwargs):
|
||||
prefix = '' if lang == '' else lang+'.'
|
||||
trn_df = pd.read_csv(self.dataset_path / f'{prefix}train.csv', header=None)
|
||||
@@ -176,10 +166,6 @@ class CLSHyperParams(LMHyperParams):
|
||||
val_len = max(int(len(trn_df) * 0.1), 2)
|
||||
trn_len = len(trn_df) - val_len
|
||||
trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:]
|
||||
trn_df = self.merge_cols(trn_df)
|
||||
val_df = self.merge_cols(val_df)
|
||||
tst_df = self.merge_cols(tst_df)
|
||||
unsup_df = self.merge_cols(unsup_df)
|
||||
kwargs.update(dict(trn_df=trn_df, val_df=val_df, tst_df=tst_df, unsup_df=unsup_df))
|
||||
return kwargs
|
||||
|
||||
|
||||
Reference in New Issue
Block a user