Note
Click here to download the full example code
DeepGOZero
This example corresponds to the paper DeepGOZero: improving protein function prediction from sequence and zero-shot learning based on ontology axioms. DeepGOZero is a machine learning model that performs protein function prediction for functions that have small number or zero annotations.
First, we have the necesary imports for this example.
import click as ck
import pandas as pd
import torch as th
import numpy as np
from torch import nn
from torch.nn import functional as F
from torch import optim
from torch.optim.lr_scheduler import MultiStepLR
from sklearn.metrics import roc_curve, auc
import math
from mowl.utils.data import FastTensorDataLoader
import os
import pickle as pkl
from tqdm import tqdm
import mowl
mowl.init_jvm("10g")
from mowl.owlapi.defaults import BOT, TOP
from mowl.datasets import ELDataset, RemoteDataset
from mowl.nn import ELEmModule
from mowl.owlapi import OWLAPIAdapter
from mowl.datasets.base import Entities, OWLClasses, OWLIndividuals
from org.semanticweb.owlapi.model import AxiomType
from org.semanticweb.owlapi.model.parameters import Imports
from org.semanticweb.owlapi.reasoner.structural import StructuralReasonerFactory
Dataset
The datasets are stored in the cloud and the following links correspond for the data for the Gene Ontology sub-ontologies: molecular function, biological process and cellular component.
MF_URL = "https://deepgo.cbrc.kaust.edu.sa/data/deepgozero/mowl/molecular_function.tar.gz"
BP_URL = "https://deepgo.cbrc.kaust.edu.sa/data/deepgozero/mowl/biological_process.tar.gz"
CC_URL = "https://deepgo.cbrc.kaust.edu.sa/data/deepgozero/mowl/cellular_component.tar.gz"
To begin, each subontology data is encapsutaled in the DGZeroDataset. This class contains three ontologies: training, validation and testing.
For this project, the training ontology is the Gene Ontology extended with the following axioms:
\(\exists has\_function. go\_class (protein)\), which encodes protein function annotations.
\(has\_interpro (protein, interpro)\), which encodes interpro features for proteins.
The validation and testing ontologies contain protein function and intepro annotations.
class DGZeroDataset(RemoteDataset):
def __init__(self, subontology):
if subontology == "mf":
url = MF_URL
root = "molecular_function/"
elif subontology == "bp":
url = BP_URL
root = "biological_process/"
elif subontology == "cc":
url = CC_URL
root = "cellular_component/"
else:
raise ValueError("Invalid subontology: {}".format(subontology))
train_owl_file = root + "mowl_train.owl"
valid_owl_file = root + "mowl_valid.owl"
test_owl_file = root + "mowl_test.owl"
super().__init__(url)
self._proteins = None
self._functions = None
self._interpros = None
@property
def functions(self):
if self._functions is None:
functions = set()
for cls_str, cls_owl in self.classes.as_dict.items():
if cls_str.startswith("http://purl.obolibrary.org/obo/GO"):
functions.add(cls_owl)
self._functions = OWLClasses(functions)
return self._functions
@property
def proteins(self):
if self._proteins is None:
proteins = set()
for ind_str, ind_owl in self.individuals.as_dict.items():
if ind_str.startswith("http://mowl/protein"):
proteins.add(ind_owl)
self._proteins = OWLIndividuals(proteins)
return self._proteins
@property
def interpros(self):
if self._interpros is None:
interpros = set()
for ind_str, ind_owl in self.individuals.as_dict.items():
if ind_str.startswith("http://mowl/interpro"):
interpros.add(ind_owl)
self._interpros = OWLIndividuals(interpros)
return self._interpros
@property
def evaluation_property(self):
return "http://mowl/has_function"
def load_data(dataset, term_to_id, ipr_to_id):
train_data = get_data(dataset.ontology, term_to_id, ipr_to_id)
valid_data = get_data(dataset.validation, term_to_id, ipr_to_id)
test_data = get_data(dataset.testing, term_to_id, ipr_to_id)
return train_data, valid_data, test_data
def get_data(ontology, term_to_id, ipr_to_id):
axioms = ontology.getABoxAxioms(Imports.fromBoolean(False))
pf_axioms = set()
interpro_axioms = set()
for abox_axiom in axioms:
ax_name = abox_axiom.getAxiomType()
if ax_name == AxiomType.CLASS_ASSERTION:
pf_axioms.add(abox_axiom)
elif ax_name == AxiomType.OBJECT_PROPERTY_ASSERTION:
interpro_axioms.add(abox_axiom)
else:
print(f"Ignoring axiom: {abox_axiom.toString()}")
individuals = ontology.getIndividualsInSignature()
proteins = [str(i.toStringID()) for i in individuals if str(i.toStringID()).startswith("http://mowl/protein/")]
proteins = sorted(proteins)
prot_to_id = {p: i for i, p in enumerate(proteins)}
data = th.zeros((len(proteins), len(ipr_to_id)), dtype=th.float32)
labels = th.zeros((len(proteins), len(term_to_id)), dtype=th.float32)
interpro_count = 0
function_count = 0
for axiom in interpro_axioms:
protein = str(axiom.getSubject().toStringID())
interpro = str(axiom.getObject().toStringID())
if interpro in ipr_to_id:
data[prot_to_id[protein], ipr_to_id[interpro]] = 1
interpro_count += 1
for axiom in pf_axioms:
protein = str(axiom.getIndividual().toStringID())
function = str(axiom.getClassExpression().getFiller().toStringID())
if function in term_to_id:
labels[prot_to_id[protein], term_to_id[function]] = 1
function_count += 1
print(f"In get_data. Interpros processed: {interpro_count}. Functions processed: {function_count}")
return data, labels
DeepGoZero model
The DeepGoZero model is composed by: - A protein encoder model that takes protein interpro features and learns a latent representation of the protein. Futhermore, this representation is associated to a GO term to predict if the GO term is a function of the protein.
class Residual(nn.Module):
def __init__(self, fn):
super().__init__()
self.fn = fn
def forward(self, x):
return x + self.fn(x)
class MLPBlock(nn.Module):
def __init__(self, in_features, out_features, bias=True, layer_norm=True, dropout=0.1, activation=nn.ReLU):
super().__init__()
self.linear = nn.Linear(in_features, out_features, bias)
self.activation = activation()
self.layer_norm = nn.BatchNorm1d(out_features) if layer_norm else None
self.dropout = nn.Dropout(dropout) if dropout else None
def forward(self, x):
x = self.activation(self.linear(x))
if self.layer_norm:
x = self.layer_norm(x)
if self.dropout:
x = self.dropout(x)
return x
The GO terms representations are learned using a model theoretic approach called ELEmbeddings. ELEmbeddings processes the axioms of the Gene Ontology and learns a representation of the GO terms.
class DGELModel(nn.Module):
def __init__(self, nb_iprs, nb_gos, nb_zero_gos, nb_rels, device, hidden_dim=1024, embed_dim=1024, margin=0.1):
super().__init__()
self.nb_gos = nb_gos
self.nb_zero_gos = nb_zero_gos
input_length = nb_iprs
net = []
net.append(MLPBlock(input_length, hidden_dim))
net.append(Residual(MLPBlock(hidden_dim, hidden_dim)))
self.net = nn.Sequential(*net)
# ELEmbeddings
self.embed_dim = embed_dim
self.hasFuncIndex = th.LongTensor([nb_rels]).to(device)
go_embed = nn.Embedding(nb_gos + nb_zero_gos+2, embed_dim)
#self.go_norm = nn.BatchNorm1d(embed_dim)
k = math.sqrt(1 / embed_dim)
nn.init.uniform_(go_embed.weight, -k, k)
go_rad = nn.Embedding(nb_gos + nb_zero_gos, 1)
nn.init.uniform_(go_rad.weight, -k, k)
rel_embed = nn.Embedding(nb_rels + 1, embed_dim)
nn.init.uniform_(rel_embed.weight, -k, k)
self.all_gos = th.arange(self.nb_gos).to(device)
self.margin = margin
self.elembeddings = ELEmModule(nb_gos + nb_zero_gos + 2, nb_rels+1, embed_dim=embed_dim) # +2 to add top and bottom
self.elembeddings.class_embed = go_embed
self.elembeddings.class_rad = go_rad
self.elembeddings.rel_embed = rel_embed
def forward(self, features, data = None):
if data is None:
data = self.all_gos
class_embed = self.elembeddings.class_embed
rel_embed = self.elembeddings.rel_embed
class_rad = self.elembeddings.class_rad
x = self.net(features)
go_embed = class_embed(data)
hasFunc = rel_embed(self.hasFuncIndex)
hasFuncGO = go_embed + hasFunc
go_rad = th.abs(class_rad(data).view(1, -1))
x = th.matmul(x, hasFuncGO.T) + go_rad
logits = th.sigmoid(x)
return logits
def predict_zero(self, features, data):
return self.forward(features, data=data)
def el_loss(self, go_normal_forms):
gci0, gci1, gci2, gci3 = go_normal_forms
gci0_loss = self.elembeddings(gci0, "gci0")
gci1_loss = self.elembeddings(gci1, "gci1")
gci2_loss = self.elembeddings(gci2, "gci2")
gci3_loss = self.elembeddings(gci3, "gci3")
return gci0_loss.mean() + gci1_loss.mean() + gci2_loss.mean() + gci3_loss.mean()
def compute_roc(labels, preds):
# Compute ROC curve and ROC area for each class
fpr, tpr, _ = roc_curve(labels.flatten(), preds.flatten())
roc_auc = auc(fpr, tpr)
return roc_auc
Training DeepGoZero
In the training phase, both the protein and GO term model are trained jointly. In the model, the objective function is composed by two terms: - The first term is the cross entropy loss between the predicted GO term and the true GO term for a protein - The second term is the ELEmbeddings loss that is computed using the axioms of the Gene Ontology
Not all the GO terms are present in the first component, but only on the second component. However, DeepGOZero is able to predict protein functions that do not have annotations by leveraging the semantics of the Gene Ontology.
def main(ont, batch_size, epochs, device):
if not os.path.exists(f"data/{ont}"):
os.makedirs(f"data/{ont}")
print("Loading DeepGOZero dataset...")
dataset = DGZeroDataset(ont)
model_file = f'data/{ont}/deepgozero_zero_10.th'
terms_file = str(dataset.root) + '/terms_zero_10.pkl'
iprs_file = str(dataset.root) + '/interpros.pkl'
out_file = str(dataset.root) + '/predictions_deepgozero_zero_10.pkl'
functions = dataset.functions.as_str
function_to_id = {f: i for i,f in enumerate(functions)}
proteins = dataset.proteins.as_str
protein_to_id = {p: i for i, p in enumerate(proteins)}
interpros = dataset.interpros.as_str
interpro_to_id = {ip: i for i, ip in enumerate(interpros)}
relations = dataset.object_properties.as_str
relation_to_id = {r: i for i, r in enumerate(relations) if r != "http://mowl/has_function"}
print(f"Functions:\t{len(functions)}")
print(f"Proteins: \t{len(proteins)}")
print(f"Interpros:\t{len(interpros)}")
print(f"Relations:\t{len(relations)}")
# List of GO terms to be used
terms_df = pd.read_pickle(terms_file)
terms = terms_df['gos'].values.flatten()
terms = ["http://purl.obolibrary.org/obo/" + t.replace(":", "_") for t in terms]
term_to_id = {t: i for i, t in enumerate(terms)}
n_terms = len(terms)
# List of Interpros to be used
ipr_df = pd.read_pickle(iprs_file)
iprs = ipr_df['interpros'].values.flatten()
iprs = ["http://mowl/interpro/" + i for i in iprs]
ipr_to_id = {v:k for k, v in enumerate(iprs)}
n_interpros = len(iprs)
print(f"GO terms list: {n_terms}")
print(f"Interpro list: {n_interpros}")
z_count = 0
z_functions = set()
for function in functions:
if not function in terms:
z_functions.add(function)
z_count += 1
print(f'Non-zero functions:\t{n_terms}\nZero functions: \t{z_count}')
zero_functions = {t: i + len(terms) for i, t in enumerate(z_functions)}
class_to_id = {**term_to_id, **zero_functions}
class_to_id[BOT] = len(class_to_id)
class_to_id[TOP] = len(class_to_id)
# Protein function data
train_data, valid_data, test_data = load_data(dataset, term_to_id, ipr_to_id)
# GO data as EL
nfs_file = f"data/{ont}/nfs.pkl"
if os.path.exists(nfs_file):
print("Loading normal forms from disk...")
with open(nfs_file, "rb") as f:
nfs = pkl.load(f)
gci0_ds, gci1_ds, gci2_ds, gci3_ds = nfs
else:
print("Generating EL dataset...")
el_dataset = ELDataset(dataset.ontology,
class_index_dict=class_to_id,
object_property_index_dict=relation_to_id,
extended=False)
nfs = el_dataset.get_gci_datasets()
with open(nfs_file, "wb") as f:
pkl.dump(nfs, f)
gci0_ds = nfs["gci0"]
gci1_ds = nfs["gci1"]
gci2_ds = nfs["gci2"]
gci3_ds = nfs["gci3"]
print(f"Axioms in GCI0: {len(gci0_ds)}")
print(f"Axioms in GCI1: {len(gci1_ds)}")
print(f"Axioms in GCI2: {len(gci2_ds)}")
print(f"Axioms in GCI3: {len(gci3_ds)}")
nfs = list(nfs.values())
n_rels = len(relation_to_id)
n_zeros = len(zero_functions)
net = DGELModel(n_interpros, n_terms, n_zeros, n_rels, device).to(device)
print(net)
train_features, train_labels = train_data
valid_features, valid_labels = valid_data
test_features, test_labels = test_data
train_loader = FastTensorDataLoader(
*train_data, batch_size=batch_size, shuffle=True)
valid_loader = FastTensorDataLoader(
*valid_data, batch_size=batch_size, shuffle=False)
test_loader = FastTensorDataLoader(
*test_data, batch_size=batch_size, shuffle=False)
valid_labels = valid_labels.detach().cpu().numpy()
test_labels = test_labels.detach().cpu().numpy()
optimizer = th.optim.Adam(net.parameters(), lr=5e-4)
scheduler = MultiStepLR(optimizer, milestones=[5, 20], gamma=0.1)
best_loss = 10000.0
print('Training the model')
for epoch in range(epochs):
net.train()
train_loss = 0
train_elloss = 0
lmbda = 0.1
train_steps = 2 # int(math.ceil(len(train_labels) / batch_size))
count = 0
for batch_features, batch_labels in tqdm(train_loader, total=train_steps):
if count == train_steps:
break
count += 1
batch_features = batch_features.to(device)
batch_labels = batch_labels.to(device)
logits = net(batch_features)
loss = F.binary_cross_entropy(logits, batch_labels)
el_loss = net.el_loss(nfs)
total_loss = loss + el_loss
train_loss += loss.detach().item()
train_elloss = el_loss.detach().item()
optimizer.zero_grad()
total_loss.backward()
optimizer.step()
train_loss /= train_steps
print('Validation')
net.eval()
with th.no_grad():
valid_steps = int(math.ceil(len(valid_labels) / batch_size))
valid_loss = 0
preds = []
for batch_features, batch_labels in tqdm(valid_loader, total=valid_steps):
batch_features = batch_features.to(device)
batch_labels = batch_labels.to(device)
logits = net(batch_features)
batch_loss = F.binary_cross_entropy(logits, batch_labels)
valid_loss += batch_loss.detach().item()
preds = np.append(preds, logits.detach().cpu().numpy())
valid_loss /= valid_steps
roc_auc = compute_roc(valid_labels, preds)
print(f'Epoch {epoch}: Loss - {train_loss}, EL Loss: {train_elloss}, Valid loss - {valid_loss}, AUC - {roc_auc}')
print('EL Loss', train_elloss)
if valid_loss < best_loss:
best_loss = valid_loss
print('Saving model')
th.save(net.state_dict(), model_file)
scheduler.step()
# Loading best model
print('Loading the best model')
net.load_state_dict(th.load(model_file))
net.eval()
with th.no_grad():
test_steps = int(math.ceil(len(test_labels) / batch_size))
test_loss = 0
preds = []
for batch_features, batch_labels in tqdm(test_loader, total=test_steps):
batch_features = batch_features.to(device)
batch_labels = batch_labels.to(device)
logits = net(batch_features)
batch_loss = F.binary_cross_entropy(logits, batch_labels)
test_loss += batch_loss.detach().cpu().item()
preds = np.append(preds, logits.detach().cpu().numpy())
test_loss /= test_steps
preds = preds.reshape(-1, n_terms)
roc_auc = compute_roc(test_labels, preds)
print(f'Test Loss - {test_loss}, AUC - {roc_auc}')
preds = list(preds)
adapter = OWLAPIAdapter()
manager = adapter.owl_manager
# Propagate scores using ontology structure
reasoner = StructuralReasonerFactory().createReasoner(dataset.ontology)
for i, scores in tqdm(enumerate(preds[:10]), total=len(preds[:10])):
prop_annots = {}
sup_processed = 0
for go_id, j in term_to_id.items():
score = scores[j]
go_class = adapter.create_class(go_id)
superclasses = reasoner.getSuperClasses(go_class, False).getFlattened()
superclasses = [str(sup.toStringID()) for sup in superclasses]
for sup_go in superclasses:
if sup_go in prop_annots:
prop_annots[sup_go] = max(prop_annots[sup_go], score)
sup_processed += 1
else:
prop_annots[sup_go] = score
for go_id, score in prop_annots.items():
if go_id in term_to_id:
scores[term_to_id[go_id]] = score
# TODO: refactor this to save predictions in an .owl file
# test_df['preds'] = preds
# test_df.to_pickle(out_file)
Training the model
ont = "mf"
batch_size = 16
epochs = 20
device = "cpu"
main(ont, batch_size, epochs, device)
Loading DeepGOZero dataset...
Functions: 50722
Proteins: 43279
Interpros: 21579
Relations: 11
GO terms list: 2041
Interpro list: 26406
Non-zero functions: 2041
Zero functions: 48681
In get_data. Interpros processed: 153955. Functions processed: 364571
In get_data. Interpros processed: 17956. Functions processed: 40176
In get_data. Interpros processed: 21084. Functions processed: 51317
Loading normal forms from disk...
Axioms in GCI0: 80941
Axioms in GCI1: 11842
Axioms in GCI2: 19594
Axioms in GCI3: 11810
DGELModel(
(net): Sequential(
(0): MLPBlock(
(linear): Linear(in_features=26406, out_features=1024, bias=True)
(activation): ReLU()
(layer_norm): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(dropout): Dropout(p=0.1, inplace=False)
)
(1): Residual(
(fn): MLPBlock(
(linear): Linear(in_features=1024, out_features=1024, bias=True)
(activation): ReLU()
(layer_norm): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(dropout): Dropout(p=0.1, inplace=False)
)
)
)
(elembeddings): ELEmModule(
(class_embed): Embedding(50724, 1024)
(class_rad): Embedding(50722, 1)
(rel_embed): Embedding(11, 1024)
)
)
Training the model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:02<00:02, 2.16s/it]
100%|##########| 2/2 [00:03<00:00, 1.82s/it]
100%|##########| 2/2 [00:03<00:00, 1.87s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 140.27it/s]
12%|#2 | 30/241 [00:00<00:01, 134.56it/s]
18%|#8 | 44/241 [00:00<00:01, 126.94it/s]
24%|##3 | 57/241 [00:00<00:01, 118.05it/s]
29%|##8 | 69/241 [00:00<00:01, 110.72it/s]
34%|###3 | 81/241 [00:00<00:01, 98.16it/s]
38%|###8 | 92/241 [00:00<00:01, 89.98it/s]
42%|####2 | 102/241 [00:01<00:01, 83.58it/s]
46%|####6 | 111/241 [00:01<00:01, 77.66it/s]
49%|####9 | 119/241 [00:01<00:01, 70.09it/s]
53%|#####2 | 127/241 [00:01<00:01, 64.91it/s]
56%|#####5 | 134/241 [00:01<00:01, 63.17it/s]
59%|#####8 | 141/241 [00:01<00:01, 62.33it/s]
61%|######1 | 148/241 [00:01<00:01, 61.07it/s]
64%|######4 | 155/241 [00:01<00:01, 59.74it/s]
67%|######6 | 161/241 [00:02<00:01, 58.55it/s]
69%|######9 | 167/241 [00:02<00:01, 57.29it/s]
72%|#######1 | 173/241 [00:02<00:01, 56.09it/s]
74%|#######4 | 179/241 [00:02<00:01, 52.86it/s]
77%|#######6 | 185/241 [00:02<00:01, 47.86it/s]
79%|#######8 | 190/241 [00:02<00:01, 47.21it/s]
81%|######## | 195/241 [00:02<00:00, 47.47it/s]
83%|########2 | 200/241 [00:02<00:00, 47.60it/s]
85%|########5 | 205/241 [00:02<00:00, 47.62it/s]
87%|########7 | 210/241 [00:03<00:00, 47.35it/s]
89%|########9 | 215/241 [00:03<00:00, 46.81it/s]
91%|#########1| 220/241 [00:03<00:00, 46.24it/s]
93%|#########3| 225/241 [00:03<00:00, 45.75it/s]
95%|#########5| 230/241 [00:03<00:00, 45.21it/s]
98%|#########7| 235/241 [00:03<00:00, 44.60it/s]
100%|#########9| 240/241 [00:03<00:00, 44.05it/s]
100%|##########| 241/241 [00:03<00:00, 63.50it/s]
Epoch 0: Loss - 0.8184013366699219, EL Loss: 8.32693099975586, Valid loss - 0.6853231515627184, AUC - 0.5034973066802776
EL Loss 8.32693099975586
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.93s/it]
100%|##########| 2/2 [00:03<00:00, 1.63s/it]
100%|##########| 2/2 [00:03<00:00, 1.68s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 141.87it/s]
12%|#2 | 30/241 [00:00<00:01, 136.80it/s]
18%|#8 | 44/241 [00:00<00:01, 132.71it/s]
24%|##4 | 58/241 [00:00<00:01, 122.92it/s]
29%|##9 | 71/241 [00:00<00:01, 120.34it/s]
35%|###4 | 84/241 [00:00<00:01, 115.99it/s]
40%|###9 | 96/241 [00:00<00:01, 112.27it/s]
45%|####4 | 108/241 [00:00<00:01, 108.57it/s]
49%|####9 | 119/241 [00:01<00:01, 105.66it/s]
54%|#####3 | 130/241 [00:01<00:01, 99.01it/s]
58%|#####8 | 140/241 [00:01<00:01, 83.20it/s]
62%|######1 | 149/241 [00:01<00:01, 70.10it/s]
65%|######5 | 157/241 [00:01<00:01, 62.78it/s]
68%|######8 | 164/241 [00:01<00:01, 58.31it/s]
71%|####### | 171/241 [00:01<00:01, 56.55it/s]
73%|#######3 | 177/241 [00:02<00:01, 55.54it/s]
76%|#######5 | 183/241 [00:02<00:01, 54.42it/s]
78%|#######8 | 189/241 [00:02<00:00, 53.30it/s]
81%|######## | 195/241 [00:02<00:00, 52.08it/s]
83%|########3 | 201/241 [00:02<00:00, 50.75it/s]
86%|########5 | 207/241 [00:02<00:00, 49.72it/s]
88%|########7 | 212/241 [00:02<00:00, 48.93it/s]
90%|######### | 217/241 [00:02<00:00, 48.20it/s]
92%|#########2| 222/241 [00:03<00:00, 47.36it/s]
94%|#########4| 227/241 [00:03<00:00, 46.65it/s]
96%|#########6| 232/241 [00:03<00:00, 45.86it/s]
98%|#########8| 237/241 [00:03<00:00, 45.15it/s]
100%|##########| 241/241 [00:03<00:00, 69.72it/s]
Epoch 1: Loss - 0.7939540445804596, EL Loss: 8.187027931213379, Valid loss - 0.6687146232335894, AUC - 0.5371056431609839
EL Loss 8.187027931213379
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:02<00:02, 2.05s/it]
100%|##########| 2/2 [00:03<00:00, 1.79s/it]
100%|##########| 2/2 [00:03<00:00, 1.83s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 140.54it/s]
12%|#2 | 30/241 [00:00<00:01, 135.64it/s]
18%|#8 | 44/241 [00:00<00:01, 130.66it/s]
24%|##4 | 58/241 [00:00<00:01, 126.52it/s]
29%|##9 | 71/241 [00:00<00:01, 122.17it/s]
35%|###4 | 84/241 [00:00<00:01, 117.40it/s]
40%|###9 | 96/241 [00:00<00:01, 112.51it/s]
45%|####4 | 108/241 [00:00<00:01, 108.81it/s]
49%|####9 | 119/241 [00:01<00:01, 105.28it/s]
54%|#####3 | 130/241 [00:01<00:01, 102.47it/s]
59%|#####8 | 141/241 [00:01<00:01, 84.56it/s]
62%|######2 | 150/241 [00:01<00:01, 75.48it/s]
66%|######5 | 158/241 [00:01<00:01, 69.57it/s]
69%|######8 | 166/241 [00:01<00:01, 59.34it/s]
72%|#######1 | 173/241 [00:01<00:01, 54.01it/s]
74%|#######4 | 179/241 [00:02<00:01, 53.53it/s]
77%|#######6 | 185/241 [00:02<00:01, 52.82it/s]
79%|#######9 | 191/241 [00:02<00:00, 51.99it/s]
82%|########1 | 197/241 [00:02<00:00, 50.11it/s]
84%|########4 | 203/241 [00:02<00:00, 46.94it/s]
86%|########6 | 208/241 [00:02<00:00, 45.95it/s]
88%|########8 | 213/241 [00:02<00:00, 45.99it/s]
90%|######### | 218/241 [00:02<00:00, 46.04it/s]
93%|#########2| 223/241 [00:03<00:00, 45.77it/s]
95%|#########4| 228/241 [00:03<00:00, 45.38it/s]
97%|#########6| 233/241 [00:03<00:00, 45.00it/s]
99%|#########8| 238/241 [00:03<00:00, 44.45it/s]
100%|##########| 241/241 [00:03<00:00, 69.27it/s]
Epoch 2: Loss - 0.8988397419452667, EL Loss: 8.049670219421387, Valid loss - 0.6525110946156671, AUC - 0.5547709801950911
EL Loss 8.049670219421387
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.80s/it]
100%|##########| 2/2 [00:03<00:00, 1.65s/it]
100%|##########| 2/2 [00:03<00:00, 1.68s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
5%|5 | 13/241 [00:00<00:01, 126.91it/s]
11%|#1 | 27/241 [00:00<00:01, 129.27it/s]
17%|#6 | 40/241 [00:00<00:01, 127.14it/s]
22%|##1 | 53/241 [00:00<00:01, 123.21it/s]
27%|##7 | 66/241 [00:00<00:01, 117.17it/s]
32%|###2 | 78/241 [00:00<00:01, 113.90it/s]
37%|###7 | 90/241 [00:00<00:01, 110.72it/s]
42%|####2 | 102/241 [00:00<00:01, 107.46it/s]
47%|####6 | 113/241 [00:01<00:01, 104.54it/s]
51%|#####1 | 124/241 [00:01<00:01, 101.14it/s]
56%|#####6 | 135/241 [00:01<00:02, 41.35it/s]
59%|#####9 | 143/241 [00:01<00:02, 44.46it/s]
63%|######2 | 151/241 [00:02<00:01, 47.03it/s]
66%|######5 | 158/241 [00:02<00:01, 46.89it/s]
68%|######8 | 165/241 [00:02<00:01, 47.50it/s]
71%|####### | 171/241 [00:02<00:01, 48.66it/s]
73%|#######3 | 177/241 [00:02<00:01, 48.63it/s]
76%|#######5 | 183/241 [00:02<00:01, 46.82it/s]
78%|#######8 | 189/241 [00:02<00:01, 46.90it/s]
80%|######## | 194/241 [00:02<00:00, 47.40it/s]
83%|########2 | 199/241 [00:03<00:00, 47.56it/s]
85%|########4 | 204/241 [00:03<00:00, 47.07it/s]
87%|########6 | 209/241 [00:03<00:00, 45.98it/s]
89%|########8 | 214/241 [00:03<00:00, 43.50it/s]
91%|######### | 219/241 [00:03<00:00, 43.05it/s]
93%|#########2| 224/241 [00:03<00:00, 43.65it/s]
95%|#########5| 229/241 [00:03<00:00, 43.86it/s]
97%|#########7| 234/241 [00:03<00:00, 43.75it/s]
99%|#########9| 239/241 [00:03<00:00, 43.56it/s]
100%|##########| 241/241 [00:04<00:00, 59.90it/s]
Epoch 3: Loss - 0.9191046059131622, EL Loss: 7.915187835693359, Valid loss - 0.6369621402495135, AUC - 0.5592231032991556
EL Loss 7.915187835693359
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.90s/it]
100%|##########| 2/2 [00:03<00:00, 1.66s/it]
100%|##########| 2/2 [00:03<00:00, 1.70s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 142.19it/s]
12%|#2 | 30/241 [00:00<00:01, 136.68it/s]
18%|#8 | 44/241 [00:00<00:01, 131.23it/s]
24%|##4 | 58/241 [00:00<00:01, 125.36it/s]
29%|##9 | 71/241 [00:00<00:01, 119.61it/s]
34%|###4 | 83/241 [00:00<00:01, 115.73it/s]
39%|###9 | 95/241 [00:00<00:01, 111.87it/s]
44%|####4 | 107/241 [00:00<00:01, 108.35it/s]
49%|####8 | 118/241 [00:01<00:01, 104.95it/s]
54%|#####3 | 129/241 [00:01<00:01, 99.84it/s]
58%|#####8 | 140/241 [00:01<00:01, 84.25it/s]
62%|######1 | 149/241 [00:01<00:01, 75.84it/s]
65%|######5 | 157/241 [00:01<00:01, 70.13it/s]
68%|######8 | 165/241 [00:01<00:01, 65.59it/s]
71%|#######1 | 172/241 [00:01<00:01, 62.23it/s]
74%|#######4 | 179/241 [00:02<00:01, 58.90it/s]
77%|#######6 | 185/241 [00:02<00:00, 56.94it/s]
79%|#######9 | 191/241 [00:02<00:00, 55.11it/s]
82%|########1 | 197/241 [00:02<00:00, 53.40it/s]
84%|########4 | 203/241 [00:02<00:00, 51.83it/s]
87%|########6 | 209/241 [00:02<00:00, 50.58it/s]
89%|########9 | 215/241 [00:02<00:00, 49.33it/s]
91%|#########1| 220/241 [00:02<00:00, 48.38it/s]
93%|#########3| 225/241 [00:02<00:00, 47.53it/s]
95%|#########5| 230/241 [00:03<00:00, 46.62it/s]
98%|#########7| 235/241 [00:03<00:00, 45.85it/s]
100%|#########9| 240/241 [00:03<00:00, 44.83it/s]
100%|##########| 241/241 [00:03<00:00, 71.67it/s]
Epoch 4: Loss - 0.8216881453990936, EL Loss: 7.783841133117676, Valid loss - 0.6290990868544677, AUC - 0.552095629859364
EL Loss 7.783841133117676
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.94s/it]
100%|##########| 2/2 [00:03<00:00, 1.67s/it]
100%|##########| 2/2 [00:03<00:00, 1.71s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|5 | 14/241 [00:00<00:01, 125.68it/s]
11%|#1 | 27/241 [00:00<00:02, 98.60it/s]
16%|#5 | 38/241 [00:00<00:02, 99.61it/s]
21%|## | 50/241 [00:00<00:01, 105.51it/s]
25%|##5 | 61/241 [00:00<00:01, 94.22it/s]
29%|##9 | 71/241 [00:00<00:01, 87.78it/s]
34%|###4 | 82/241 [00:00<00:01, 92.44it/s]
39%|###8 | 93/241 [00:00<00:01, 94.87it/s]
43%|####2 | 103/241 [00:01<00:01, 93.49it/s]
47%|####6 | 113/241 [00:01<00:01, 82.78it/s]
51%|##### | 122/241 [00:01<00:01, 81.98it/s]
54%|#####4 | 131/241 [00:01<00:01, 82.81it/s]
58%|#####8 | 140/241 [00:01<00:01, 74.50it/s]
61%|######1 | 148/241 [00:01<00:01, 69.11it/s]
65%|######4 | 156/241 [00:01<00:01, 64.81it/s]
68%|######7 | 163/241 [00:02<00:01, 60.02it/s]
71%|####### | 170/241 [00:02<00:01, 54.92it/s]
73%|#######3 | 176/241 [00:02<00:01, 52.53it/s]
76%|#######5 | 182/241 [00:02<00:01, 51.88it/s]
78%|#######8 | 188/241 [00:02<00:01, 51.14it/s]
80%|######## | 194/241 [00:02<00:00, 50.51it/s]
83%|########2 | 200/241 [00:02<00:00, 49.80it/s]
85%|########5 | 205/241 [00:02<00:00, 49.14it/s]
87%|########7 | 210/241 [00:03<00:00, 48.41it/s]
89%|########9 | 215/241 [00:03<00:00, 47.53it/s]
91%|#########1| 220/241 [00:03<00:00, 46.93it/s]
93%|#########3| 225/241 [00:03<00:00, 46.24it/s]
95%|#########5| 230/241 [00:03<00:00, 45.59it/s]
98%|#########7| 235/241 [00:03<00:00, 44.85it/s]
100%|#########9| 240/241 [00:03<00:00, 44.19it/s]
100%|##########| 241/241 [00:03<00:00, 64.84it/s]
Epoch 5: Loss - 0.8393794298171997, EL Loss: 7.712863922119141, Valid loss - 0.625623598880293, AUC - 0.5492142567511584
EL Loss 7.712863922119141
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.91s/it]
100%|##########| 2/2 [00:03<00:00, 1.67s/it]
100%|##########| 2/2 [00:03<00:00, 1.70s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
5%|5 | 13/241 [00:00<00:01, 126.85it/s]
11%|# | 26/241 [00:00<00:01, 126.12it/s]
16%|#6 | 39/241 [00:00<00:01, 125.48it/s]
22%|##1 | 52/241 [00:00<00:01, 123.24it/s]
27%|##6 | 65/241 [00:00<00:01, 118.80it/s]
32%|###1 | 77/241 [00:00<00:01, 114.61it/s]
37%|###6 | 89/241 [00:00<00:01, 110.80it/s]
42%|####1 | 101/241 [00:00<00:01, 107.20it/s]
46%|####6 | 112/241 [00:00<00:01, 104.23it/s]
51%|#####1 | 123/241 [00:01<00:01, 101.06it/s]
56%|#####5 | 134/241 [00:01<00:01, 89.39it/s]
60%|#####9 | 144/241 [00:01<00:01, 78.37it/s]
63%|######3 | 153/241 [00:01<00:01, 71.32it/s]
67%|######6 | 161/241 [00:01<00:01, 66.50it/s]
70%|######9 | 168/241 [00:01<00:01, 62.94it/s]
73%|#######2 | 175/241 [00:02<00:01, 59.29it/s]
76%|#######5 | 182/241 [00:02<00:01, 56.65it/s]
78%|#######8 | 188/241 [00:02<00:00, 54.91it/s]
80%|######## | 194/241 [00:02<00:00, 53.24it/s]
83%|########2 | 200/241 [00:02<00:00, 51.68it/s]
85%|########5 | 206/241 [00:02<00:00, 50.13it/s]
88%|########7 | 212/241 [00:02<00:00, 48.19it/s]
90%|######### | 217/241 [00:02<00:00, 47.60it/s]
92%|#########2| 222/241 [00:03<00:00, 46.89it/s]
94%|#########4| 227/241 [00:03<00:00, 46.23it/s]
96%|#########6| 232/241 [00:03<00:00, 45.37it/s]
98%|#########8| 237/241 [00:03<00:00, 44.74it/s]
100%|##########| 241/241 [00:03<00:00, 70.01it/s]
Epoch 6: Loss - 0.8023559749126434, EL Loss: 7.700018882751465, Valid loss - 0.6257101638683145, AUC - 0.5477172213355197
EL Loss 7.700018882751465
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.79s/it]
100%|##########| 2/2 [00:03<00:00, 1.54s/it]
100%|##########| 2/2 [00:03<00:00, 1.58s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 140.97it/s]
12%|#2 | 30/241 [00:00<00:01, 136.89it/s]
18%|#8 | 44/241 [00:00<00:01, 131.04it/s]
24%|##4 | 58/241 [00:00<00:01, 125.80it/s]
29%|##9 | 71/241 [00:00<00:01, 119.22it/s]
34%|###4 | 83/241 [00:00<00:01, 115.80it/s]
39%|###9 | 95/241 [00:00<00:01, 111.76it/s]
44%|####4 | 107/241 [00:00<00:01, 108.51it/s]
49%|####8 | 118/241 [00:01<00:01, 104.98it/s]
54%|#####3 | 129/241 [00:01<00:01, 102.51it/s]
58%|#####8 | 140/241 [00:01<00:01, 84.75it/s]
62%|######1 | 149/241 [00:01<00:01, 75.91it/s]
65%|######5 | 157/241 [00:01<00:01, 69.93it/s]
68%|######8 | 165/241 [00:01<00:01, 65.17it/s]
71%|#######1 | 172/241 [00:01<00:01, 61.69it/s]
74%|#######4 | 179/241 [00:02<00:01, 58.81it/s]
77%|#######6 | 185/241 [00:02<00:00, 56.64it/s]
79%|#######9 | 191/241 [00:02<00:00, 54.74it/s]
82%|########1 | 197/241 [00:02<00:00, 53.02it/s]
84%|########4 | 203/241 [00:02<00:00, 51.55it/s]
87%|########6 | 209/241 [00:02<00:00, 50.21it/s]
89%|########9 | 215/241 [00:02<00:00, 48.76it/s]
91%|#########1| 220/241 [00:02<00:00, 47.73it/s]
93%|#########3| 225/241 [00:03<00:00, 46.96it/s]
95%|#########5| 230/241 [00:03<00:00, 46.19it/s]
98%|#########7| 235/241 [00:03<00:00, 45.48it/s]
100%|#########9| 240/241 [00:03<00:00, 44.80it/s]
100%|##########| 241/241 [00:03<00:00, 71.46it/s]
Epoch 7: Loss - 0.7494914531707764, EL Loss: 7.687170505523682, Valid loss - 0.6205516813701614, AUC - 0.546431852647717
EL Loss 7.687170505523682
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.96s/it]
100%|##########| 2/2 [00:03<00:00, 1.71s/it]
100%|##########| 2/2 [00:03<00:00, 1.74s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 140.45it/s]
12%|#2 | 30/241 [00:00<00:01, 135.79it/s]
18%|#8 | 44/241 [00:00<00:01, 129.39it/s]
24%|##3 | 57/241 [00:00<00:01, 124.32it/s]
29%|##9 | 70/241 [00:00<00:01, 118.26it/s]
34%|###4 | 82/241 [00:00<00:01, 115.00it/s]
39%|###9 | 94/241 [00:00<00:01, 111.26it/s]
44%|####3 | 106/241 [00:00<00:01, 108.17it/s]
49%|####8 | 117/241 [00:01<00:02, 45.22it/s]
52%|#####2 | 126/241 [00:01<00:02, 51.07it/s]
56%|#####6 | 135/241 [00:01<00:01, 53.61it/s]
59%|#####9 | 143/241 [00:01<00:01, 55.15it/s]
63%|######2 | 151/241 [00:02<00:01, 55.52it/s]
66%|######5 | 158/241 [00:02<00:01, 55.20it/s]
68%|######8 | 165/241 [00:02<00:01, 55.22it/s]
71%|#######1 | 172/241 [00:02<00:01, 54.82it/s]
74%|#######3 | 178/241 [00:02<00:01, 54.30it/s]
76%|#######6 | 184/241 [00:02<00:01, 53.56it/s]
79%|#######8 | 190/241 [00:02<00:00, 52.68it/s]
81%|########1 | 196/241 [00:02<00:00, 51.66it/s]
84%|########3 | 202/241 [00:03<00:00, 50.68it/s]
86%|########6 | 208/241 [00:03<00:00, 49.84it/s]
89%|########8 | 214/241 [00:03<00:00, 48.93it/s]
91%|######### | 219/241 [00:03<00:00, 48.23it/s]
93%|#########2| 224/241 [00:03<00:00, 47.47it/s]
95%|#########5| 229/241 [00:03<00:00, 46.75it/s]
97%|#########7| 234/241 [00:03<00:00, 45.94it/s]
99%|#########9| 239/241 [00:03<00:00, 45.22it/s]
100%|##########| 241/241 [00:03<00:00, 62.20it/s]
Epoch 8: Loss - 0.8320796489715576, EL Loss: 7.674323081970215, Valid loss - 0.6145079403002727, AUC - 0.5428175356329648
EL Loss 7.674323081970215
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.83s/it]
100%|##########| 2/2 [00:03<00:00, 1.56s/it]
100%|##########| 2/2 [00:03<00:00, 1.60s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 140.60it/s]
12%|#2 | 30/241 [00:00<00:01, 127.81it/s]
18%|#7 | 43/241 [00:00<00:01, 124.48it/s]
23%|##3 | 56/241 [00:00<00:01, 120.13it/s]
29%|##8 | 69/241 [00:00<00:01, 116.54it/s]
34%|###3 | 81/241 [00:00<00:01, 113.21it/s]
39%|###8 | 93/241 [00:00<00:01, 110.61it/s]
44%|####3 | 105/241 [00:00<00:01, 108.10it/s]
48%|####8 | 116/241 [00:01<00:01, 105.95it/s]
53%|#####2 | 127/241 [00:01<00:01, 103.09it/s]
57%|#####7 | 138/241 [00:01<00:01, 86.43it/s]
61%|######1 | 148/241 [00:01<00:01, 76.73it/s]
65%|######5 | 157/241 [00:01<00:01, 70.45it/s]
68%|######8 | 165/241 [00:01<00:01, 65.94it/s]
71%|#######1 | 172/241 [00:01<00:01, 62.58it/s]
74%|#######4 | 179/241 [00:02<00:01, 59.74it/s]
77%|#######7 | 186/241 [00:02<00:00, 57.30it/s]
80%|#######9 | 192/241 [00:02<00:00, 55.29it/s]
82%|########2 | 198/241 [00:02<00:00, 53.53it/s]
85%|########4 | 204/241 [00:02<00:00, 51.99it/s]
87%|########7 | 210/241 [00:02<00:00, 50.67it/s]
90%|########9 | 216/241 [00:02<00:00, 49.43it/s]
92%|#########1| 221/241 [00:02<00:00, 48.49it/s]
94%|#########3| 226/241 [00:03<00:00, 47.58it/s]
96%|#########5| 231/241 [00:03<00:00, 46.69it/s]
98%|#########7| 236/241 [00:03<00:00, 45.86it/s]
100%|##########| 241/241 [00:03<00:00, 44.77it/s]
100%|##########| 241/241 [00:03<00:00, 71.51it/s]
Epoch 9: Loss - 0.7844649851322174, EL Loss: 7.661462783813477, Valid loss - 0.6104145589211175, AUC - 0.5435321699907602
EL Loss 7.661462783813477
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:02<00:02, 2.00s/it]
100%|##########| 2/2 [00:03<00:00, 1.70s/it]
100%|##########| 2/2 [00:03<00:00, 1.75s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 144.93it/s]
12%|#2 | 30/241 [00:00<00:01, 140.30it/s]
19%|#8 | 45/241 [00:00<00:01, 132.93it/s]
24%|##4 | 59/241 [00:00<00:01, 127.74it/s]
30%|##9 | 72/241 [00:00<00:01, 120.97it/s]
35%|###5 | 85/241 [00:00<00:01, 117.11it/s]
40%|#### | 97/241 [00:00<00:01, 113.52it/s]
45%|####5 | 109/241 [00:00<00:01, 110.38it/s]
50%|##### | 121/241 [00:01<00:01, 106.96it/s]
55%|#####4 | 132/241 [00:01<00:01, 99.36it/s]
59%|#####9 | 143/241 [00:01<00:01, 83.94it/s]
63%|######3 | 152/241 [00:01<00:01, 75.42it/s]
66%|######6 | 160/241 [00:01<00:01, 69.68it/s]
70%|######9 | 168/241 [00:01<00:01, 65.13it/s]
73%|#######2 | 175/241 [00:01<00:01, 61.79it/s]
76%|#######5 | 182/241 [00:02<00:00, 59.01it/s]
78%|#######8 | 188/241 [00:02<00:00, 56.87it/s]
80%|######## | 194/241 [00:02<00:00, 54.87it/s]
83%|########2 | 200/241 [00:02<00:00, 53.17it/s]
85%|########5 | 206/241 [00:02<00:00, 51.69it/s]
88%|########7 | 212/241 [00:02<00:00, 50.16it/s]
90%|######### | 218/241 [00:02<00:00, 49.05it/s]
93%|#########2| 223/241 [00:02<00:00, 48.14it/s]
95%|#########4| 228/241 [00:03<00:00, 47.31it/s]
97%|#########6| 233/241 [00:03<00:00, 46.51it/s]
99%|#########8| 238/241 [00:03<00:00, 45.75it/s]
100%|##########| 241/241 [00:03<00:00, 72.62it/s]
Epoch 10: Loss - 0.8245735764503479, EL Loss: 7.648594856262207, Valid loss - 0.6103225924167396, AUC - 0.543750684069164
EL Loss 7.648594856262207
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.79s/it]
100%|##########| 2/2 [00:03<00:00, 1.55s/it]
100%|##########| 2/2 [00:03<00:00, 1.59s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 144.94it/s]
12%|#2 | 30/241 [00:00<00:01, 138.78it/s]
18%|#8 | 44/241 [00:00<00:01, 133.48it/s]
24%|##4 | 58/241 [00:00<00:01, 127.86it/s]
29%|##9 | 71/241 [00:00<00:01, 121.07it/s]
35%|###4 | 84/241 [00:00<00:01, 116.41it/s]
40%|###9 | 96/241 [00:00<00:01, 112.62it/s]
45%|####4 | 108/241 [00:00<00:01, 108.66it/s]
49%|####9 | 119/241 [00:01<00:01, 105.77it/s]
54%|#####3 | 130/241 [00:01<00:01, 99.04it/s]
58%|#####8 | 140/241 [00:01<00:01, 84.80it/s]
62%|######1 | 149/241 [00:01<00:01, 76.16it/s]
65%|######5 | 157/241 [00:01<00:01, 70.31it/s]
68%|######8 | 165/241 [00:01<00:01, 65.55it/s]
71%|#######1 | 172/241 [00:01<00:01, 62.10it/s]
74%|#######4 | 179/241 [00:02<00:01, 59.20it/s]
77%|#######6 | 185/241 [00:02<00:00, 56.80it/s]
79%|#######9 | 191/241 [00:02<00:00, 54.95it/s]
82%|########1 | 197/241 [00:02<00:00, 53.30it/s]
84%|########4 | 203/241 [00:02<00:00, 51.84it/s]
87%|########6 | 209/241 [00:02<00:00, 50.58it/s]
89%|########9 | 215/241 [00:02<00:00, 49.41it/s]
91%|#########1| 220/241 [00:02<00:00, 48.32it/s]
93%|#########3| 225/241 [00:02<00:00, 47.56it/s]
95%|#########5| 230/241 [00:03<00:00, 46.69it/s]
98%|#########7| 235/241 [00:03<00:00, 45.86it/s]
100%|#########9| 240/241 [00:03<00:00, 45.12it/s]
100%|##########| 241/241 [00:03<00:00, 72.00it/s]
Epoch 11: Loss - 0.777436763048172, EL Loss: 7.635721206665039, Valid loss - 0.607888503688005, AUC - 0.5429795159827281
EL Loss 7.635721206665039
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.91s/it]
100%|##########| 2/2 [00:03<00:00, 1.68s/it]
100%|##########| 2/2 [00:03<00:00, 1.71s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 144.90it/s]
12%|#2 | 30/241 [00:00<00:01, 140.30it/s]
19%|#8 | 45/241 [00:00<00:01, 134.49it/s]
24%|##4 | 59/241 [00:00<00:01, 128.88it/s]
30%|##9 | 72/241 [00:00<00:01, 122.61it/s]
35%|###5 | 85/241 [00:00<00:01, 117.40it/s]
40%|#### | 97/241 [00:00<00:01, 112.87it/s]
45%|####5 | 109/241 [00:00<00:01, 109.11it/s]
50%|####9 | 120/241 [00:01<00:01, 104.44it/s]
54%|#####4 | 131/241 [00:01<00:01, 100.25it/s]
59%|#####8 | 142/241 [00:01<00:01, 84.44it/s]
63%|######2 | 151/241 [00:01<00:01, 75.79it/s]
66%|######5 | 159/241 [00:01<00:01, 70.00it/s]
69%|######9 | 167/241 [00:01<00:01, 65.42it/s]
72%|#######2 | 174/241 [00:01<00:01, 62.05it/s]
75%|#######5 | 181/241 [00:02<00:01, 59.14it/s]
78%|#######7 | 187/241 [00:02<00:00, 57.08it/s]
80%|######## | 193/241 [00:02<00:00, 55.15it/s]
83%|########2 | 199/241 [00:02<00:00, 53.40it/s]
85%|########5 | 205/241 [00:02<00:00, 51.86it/s]
88%|########7 | 211/241 [00:02<00:00, 50.47it/s]
90%|######### | 217/241 [00:02<00:00, 49.31it/s]
92%|#########2| 222/241 [00:02<00:00, 48.32it/s]
94%|#########4| 227/241 [00:02<00:00, 47.47it/s]
96%|#########6| 232/241 [00:03<00:00, 46.60it/s]
98%|#########8| 237/241 [00:03<00:00, 45.70it/s]
100%|##########| 241/241 [00:03<00:00, 72.56it/s]
Epoch 12: Loss - 0.8953540921211243, EL Loss: 7.62283992767334, Valid loss - 0.6025691368767829, AUC - 0.5426237278146973
EL Loss 7.62283992767334
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.81s/it]
100%|##########| 2/2 [00:03<00:00, 1.57s/it]
100%|##########| 2/2 [00:03<00:00, 1.60s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 144.15it/s]
12%|#2 | 30/241 [00:00<00:01, 132.48it/s]
18%|#8 | 44/241 [00:00<00:01, 130.80it/s]
24%|##4 | 58/241 [00:00<00:01, 127.17it/s]
29%|##9 | 71/241 [00:00<00:01, 121.73it/s]
35%|###4 | 84/241 [00:00<00:01, 118.30it/s]
40%|###9 | 96/241 [00:00<00:01, 115.14it/s]
45%|####4 | 108/241 [00:01<00:02, 46.98it/s]
49%|####8 | 118/241 [00:01<00:02, 54.14it/s]
53%|#####3 | 128/241 [00:01<00:01, 61.53it/s]
57%|#####7 | 138/241 [00:01<00:01, 62.74it/s]
61%|###### | 147/241 [00:01<00:01, 61.70it/s]
64%|######4 | 155/241 [00:02<00:01, 60.51it/s]
68%|######7 | 163/241 [00:02<00:01, 59.13it/s]
71%|####### | 170/241 [00:02<00:01, 57.78it/s]
73%|#######3 | 177/241 [00:02<00:01, 56.39it/s]
76%|#######5 | 183/241 [00:02<00:01, 55.14it/s]
78%|#######8 | 189/241 [00:02<00:00, 53.94it/s]
81%|######## | 195/241 [00:02<00:00, 52.81it/s]
83%|########3 | 201/241 [00:02<00:00, 51.63it/s]
86%|########5 | 207/241 [00:03<00:00, 50.48it/s]
88%|########8 | 213/241 [00:03<00:00, 49.45it/s]
90%|######### | 218/241 [00:03<00:00, 48.66it/s]
93%|#########2| 223/241 [00:03<00:00, 47.32it/s]
95%|#########4| 228/241 [00:03<00:00, 46.57it/s]
97%|#########6| 233/241 [00:03<00:00, 45.87it/s]
99%|#########8| 238/241 [00:03<00:00, 45.07it/s]
100%|##########| 241/241 [00:03<00:00, 63.06it/s]
Epoch 13: Loss - 0.8028044700622559, EL Loss: 7.6099467277526855, Valid loss - 0.6036613034509525, AUC - 0.5428270105417972
EL Loss 7.6099467277526855
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.87s/it]
100%|##########| 2/2 [00:03<00:00, 1.65s/it]
100%|##########| 2/2 [00:03<00:00, 1.68s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 141.96it/s]
12%|#2 | 30/241 [00:00<00:01, 136.66it/s]
18%|#8 | 44/241 [00:00<00:01, 130.33it/s]
24%|##4 | 58/241 [00:00<00:01, 125.20it/s]
29%|##9 | 71/241 [00:00<00:01, 119.07it/s]
34%|###4 | 83/241 [00:00<00:01, 115.09it/s]
39%|###9 | 95/241 [00:00<00:01, 111.85it/s]
44%|####4 | 107/241 [00:00<00:01, 108.43it/s]
49%|####8 | 118/241 [00:01<00:01, 105.31it/s]
54%|#####3 | 129/241 [00:01<00:01, 102.52it/s]
58%|#####8 | 140/241 [00:01<00:01, 84.80it/s]
62%|######1 | 149/241 [00:01<00:01, 75.60it/s]
65%|######5 | 157/241 [00:01<00:01, 69.70it/s]
68%|######8 | 165/241 [00:01<00:01, 64.93it/s]
71%|#######1 | 172/241 [00:01<00:01, 61.65it/s]
74%|#######4 | 179/241 [00:02<00:01, 58.61it/s]
77%|#######6 | 185/241 [00:02<00:00, 56.56it/s]
79%|#######9 | 191/241 [00:02<00:00, 54.68it/s]
82%|########1 | 197/241 [00:02<00:00, 53.04it/s]
84%|########4 | 203/241 [00:02<00:00, 51.63it/s]
87%|########6 | 209/241 [00:02<00:00, 50.32it/s]
89%|########9 | 215/241 [00:02<00:00, 48.98it/s]
91%|#########1| 220/241 [00:02<00:00, 48.13it/s]
93%|#########3| 225/241 [00:03<00:00, 47.28it/s]
95%|#########5| 230/241 [00:03<00:00, 46.45it/s]
98%|#########7| 235/241 [00:03<00:00, 45.67it/s]
100%|#########9| 240/241 [00:03<00:00, 44.92it/s]
100%|##########| 241/241 [00:03<00:00, 71.49it/s]
Epoch 14: Loss - 0.7360684871673584, EL Loss: 7.597043991088867, Valid loss - 0.5949995933231971, AUC - 0.5427104708009962
EL Loss 7.597043991088867
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.78s/it]
100%|##########| 2/2 [00:03<00:00, 1.55s/it]
100%|##########| 2/2 [00:03<00:00, 1.59s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 140.97it/s]
12%|#2 | 30/241 [00:00<00:01, 136.38it/s]
18%|#8 | 44/241 [00:00<00:01, 131.82it/s]
24%|##4 | 58/241 [00:00<00:01, 127.13it/s]
29%|##9 | 71/241 [00:00<00:01, 122.46it/s]
35%|###4 | 84/241 [00:00<00:01, 116.91it/s]
40%|###9 | 96/241 [00:00<00:01, 113.07it/s]
45%|####4 | 108/241 [00:00<00:01, 109.33it/s]
49%|####9 | 119/241 [00:01<00:01, 106.32it/s]
54%|#####3 | 130/241 [00:01<00:01, 99.26it/s]
58%|#####8 | 140/241 [00:01<00:01, 84.47it/s]
62%|######1 | 149/241 [00:01<00:01, 75.74it/s]
65%|######5 | 157/241 [00:01<00:01, 69.85it/s]
68%|######8 | 165/241 [00:01<00:01, 65.24it/s]
71%|#######1 | 172/241 [00:01<00:01, 61.89it/s]
74%|#######4 | 179/241 [00:02<00:01, 58.77it/s]
77%|#######6 | 185/241 [00:02<00:00, 56.77it/s]
79%|#######9 | 191/241 [00:02<00:00, 54.92it/s]
82%|########1 | 197/241 [00:02<00:00, 53.21it/s]
84%|########4 | 203/241 [00:02<00:00, 51.68it/s]
87%|########6 | 209/241 [00:02<00:00, 50.43it/s]
89%|########9 | 215/241 [00:02<00:00, 49.26it/s]
91%|#########1| 220/241 [00:02<00:00, 48.35it/s]
93%|#########3| 225/241 [00:02<00:00, 47.54it/s]
95%|#########5| 230/241 [00:03<00:00, 46.66it/s]
98%|#########7| 235/241 [00:03<00:00, 45.81it/s]
100%|#########9| 240/241 [00:03<00:00, 45.01it/s]
100%|##########| 241/241 [00:03<00:00, 71.85it/s]
Epoch 15: Loss - 0.82499960064888, EL Loss: 7.584137916564941, Valid loss - 0.5945237669707334, AUC - 0.5430325626362527
EL Loss 7.584137916564941
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.90s/it]
100%|##########| 2/2 [00:03<00:00, 1.65s/it]
100%|##########| 2/2 [00:03<00:00, 1.69s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 145.22it/s]
12%|#2 | 30/241 [00:00<00:01, 139.74it/s]
18%|#8 | 44/241 [00:00<00:01, 134.20it/s]
24%|##4 | 58/241 [00:00<00:01, 129.51it/s]
29%|##9 | 71/241 [00:00<00:01, 124.53it/s]
35%|###4 | 84/241 [00:00<00:01, 119.51it/s]
40%|###9 | 96/241 [00:00<00:01, 115.65it/s]
45%|####4 | 108/241 [00:00<00:01, 111.78it/s]
50%|####9 | 120/241 [00:01<00:01, 108.31it/s]
54%|#####4 | 131/241 [00:01<00:01, 100.14it/s]
59%|#####8 | 142/241 [00:01<00:01, 84.66it/s]
63%|######2 | 151/241 [00:01<00:01, 76.25it/s]
66%|######5 | 159/241 [00:01<00:01, 70.48it/s]
69%|######9 | 167/241 [00:01<00:01, 65.85it/s]
72%|#######2 | 174/241 [00:01<00:01, 62.50it/s]
75%|#######5 | 181/241 [00:02<00:01, 59.68it/s]
78%|#######8 | 188/241 [00:02<00:00, 57.22it/s]
80%|######## | 194/241 [00:02<00:00, 55.31it/s]
83%|########2 | 200/241 [00:02<00:00, 53.57it/s]
85%|########5 | 206/241 [00:02<00:00, 52.01it/s]
88%|########7 | 212/241 [00:02<00:00, 50.65it/s]
90%|######### | 218/241 [00:02<00:00, 49.47it/s]
93%|#########2| 223/241 [00:02<00:00, 48.49it/s]
95%|#########4| 228/241 [00:03<00:00, 47.53it/s]
97%|#########6| 233/241 [00:03<00:00, 46.71it/s]
99%|#########8| 238/241 [00:03<00:00, 45.81it/s]
100%|##########| 241/241 [00:03<00:00, 73.05it/s]
Epoch 16: Loss - 0.7356415390968323, EL Loss: 7.571218490600586, Valid loss - 0.5899853904217606, AUC - 0.542542484996196
EL Loss 7.571218490600586
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.78s/it]
100%|##########| 2/2 [00:03<00:00, 1.55s/it]
100%|##########| 2/2 [00:03<00:00, 1.59s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|6 | 15/241 [00:00<00:01, 145.09it/s]
12%|#2 | 30/241 [00:00<00:01, 137.87it/s]
18%|#8 | 44/241 [00:00<00:01, 131.20it/s]
24%|##4 | 58/241 [00:00<00:01, 125.03it/s]
29%|##9 | 71/241 [00:00<00:01, 120.83it/s]
35%|###4 | 84/241 [00:00<00:01, 116.62it/s]
40%|###9 | 96/241 [00:00<00:01, 112.67it/s]
45%|####4 | 108/241 [00:00<00:01, 109.04it/s]
49%|####9 | 119/241 [00:01<00:01, 106.00it/s]
54%|#####3 | 130/241 [00:01<00:01, 98.94it/s]
58%|#####8 | 140/241 [00:01<00:01, 84.73it/s]
62%|######1 | 149/241 [00:01<00:01, 76.09it/s]
65%|######5 | 157/241 [00:01<00:01, 70.30it/s]
68%|######8 | 165/241 [00:01<00:01, 65.59it/s]
71%|#######1 | 172/241 [00:01<00:01, 62.26it/s]
74%|#######4 | 179/241 [00:02<00:01, 59.46it/s]
77%|#######7 | 186/241 [00:02<00:00, 57.08it/s]
80%|#######9 | 192/241 [00:02<00:00, 55.20it/s]
82%|########2 | 198/241 [00:02<00:00, 53.52it/s]
85%|########4 | 204/241 [00:02<00:00, 51.91it/s]
87%|########7 | 210/241 [00:02<00:00, 50.71it/s]
90%|########9 | 216/241 [00:02<00:00, 49.53it/s]
92%|#########1| 221/241 [00:02<00:00, 48.61it/s]
94%|#########3| 226/241 [00:02<00:00, 47.72it/s]
96%|#########5| 231/241 [00:03<00:00, 46.86it/s]
98%|#########7| 236/241 [00:03<00:00, 45.48it/s]
100%|##########| 241/241 [00:03<00:00, 44.51it/s]
100%|##########| 241/241 [00:03<00:00, 72.06it/s]
Epoch 17: Loss - 0.7992963790893555, EL Loss: 7.558291912078857, Valid loss - 0.5880378353645198, AUC - 0.5422376921463499
EL Loss 7.558291912078857
Saving model
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.89s/it]
100%|##########| 2/2 [00:03<00:00, 1.65s/it]
100%|##########| 2/2 [00:03<00:00, 1.69s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|5 | 14/241 [00:00<00:01, 137.62it/s]
12%|#1 | 28/241 [00:00<00:01, 133.05it/s]
17%|#7 | 42/241 [00:00<00:01, 126.63it/s]
23%|##2 | 55/241 [00:00<00:01, 122.01it/s]
28%|##8 | 68/241 [00:00<00:01, 118.15it/s]
33%|###3 | 80/241 [00:00<00:01, 114.27it/s]
38%|###8 | 92/241 [00:01<00:03, 46.52it/s]
42%|####2 | 102/241 [00:01<00:02, 54.04it/s]
46%|####6 | 112/241 [00:01<00:02, 61.77it/s]
51%|##### | 122/241 [00:01<00:01, 68.60it/s]
55%|#####4 | 132/241 [00:01<00:01, 70.31it/s]
59%|#####8 | 141/241 [00:01<00:01, 66.54it/s]
62%|######1 | 149/241 [00:02<00:01, 63.50it/s]
65%|######5 | 157/241 [00:02<00:01, 61.33it/s]
68%|######8 | 164/241 [00:02<00:01, 59.42it/s]
71%|####### | 171/241 [00:02<00:01, 57.69it/s]
74%|#######3 | 178/241 [00:02<00:01, 55.90it/s]
76%|#######6 | 184/241 [00:02<00:01, 54.63it/s]
79%|#######8 | 190/241 [00:02<00:00, 53.34it/s]
81%|########1 | 196/241 [00:02<00:00, 52.03it/s]
84%|########3 | 202/241 [00:03<00:00, 50.87it/s]
86%|########6 | 208/241 [00:03<00:00, 49.75it/s]
88%|########8 | 213/241 [00:03<00:00, 48.82it/s]
90%|######### | 218/241 [00:03<00:00, 47.98it/s]
93%|#########2| 223/241 [00:03<00:00, 47.13it/s]
95%|#########4| 228/241 [00:03<00:00, 46.33it/s]
97%|#########6| 233/241 [00:03<00:00, 45.64it/s]
99%|#########8| 238/241 [00:03<00:00, 44.86it/s]
100%|##########| 241/241 [00:03<00:00, 61.78it/s]
Epoch 18: Loss - 0.7941900789737701, EL Loss: 7.545344829559326, Valid loss - 0.593473576411172, AUC - 0.5399753424719194
EL Loss 7.545344829559326
0%| | 0/2 [00:00<?, ?it/s]
50%|##### | 1/2 [00:01<00:01, 1.76s/it]
100%|##########| 2/2 [00:03<00:00, 1.53s/it]
100%|##########| 2/2 [00:03<00:00, 1.57s/it]
Validation
0%| | 0/241 [00:00<?, ?it/s]
6%|5 | 14/241 [00:00<00:01, 138.49it/s]
12%|#1 | 28/241 [00:00<00:01, 135.05it/s]
17%|#7 | 42/241 [00:00<00:01, 108.66it/s]
22%|##2 | 54/241 [00:00<00:01, 96.48it/s]
27%|##6 | 65/241 [00:00<00:01, 98.47it/s]
32%|###1 | 76/241 [00:00<00:01, 101.77it/s]
36%|###6 | 87/241 [00:00<00:01, 102.88it/s]
41%|#### | 98/241 [00:00<00:01, 100.63it/s]
45%|####5 | 109/241 [00:01<00:01, 96.39it/s]
49%|####9 | 119/241 [00:01<00:01, 92.65it/s]
54%|#####3 | 129/241 [00:01<00:01, 84.17it/s]
57%|#####7 | 138/241 [00:01<00:01, 75.62it/s]
61%|###### | 146/241 [00:01<00:01, 70.08it/s]
64%|######3 | 154/241 [00:01<00:01, 65.87it/s]
67%|######6 | 161/241 [00:01<00:01, 62.78it/s]
70%|######9 | 168/241 [00:02<00:01, 60.07it/s]
73%|#######2 | 175/241 [00:02<00:01, 57.72it/s]
75%|#######5 | 181/241 [00:02<00:01, 55.94it/s]
78%|#######7 | 187/241 [00:02<00:00, 54.15it/s]
80%|######## | 193/241 [00:02<00:00, 52.66it/s]
83%|########2 | 199/241 [00:02<00:00, 51.38it/s]
85%|########5 | 205/241 [00:02<00:00, 50.18it/s]
88%|########7 | 211/241 [00:02<00:00, 49.19it/s]
90%|########9 | 216/241 [00:03<00:00, 48.31it/s]
92%|#########1| 221/241 [00:03<00:00, 47.39it/s]
94%|#########3| 226/241 [00:03<00:00, 45.89it/s]
96%|#########5| 231/241 [00:03<00:00, 44.22it/s]
98%|#########7| 236/241 [00:03<00:00, 42.83it/s]
100%|##########| 241/241 [00:03<00:00, 41.41it/s]
100%|##########| 241/241 [00:03<00:00, 66.77it/s]
Epoch 19: Loss - 0.7346799671649933, EL Loss: 7.532385349273682, Valid loss - 0.5922719287179813, AUC - 0.5404216459528635
EL Loss 7.532385349273682
Loading the best model
0%| | 0/295 [00:00<?, ?it/s]
5%|4 | 14/295 [00:00<00:02, 136.03it/s]
9%|9 | 28/295 [00:00<00:02, 133.33it/s]
14%|#4 | 42/295 [00:00<00:01, 126.71it/s]
19%|#8 | 55/295 [00:00<00:01, 123.00it/s]
23%|##3 | 68/295 [00:00<00:01, 119.54it/s]
27%|##7 | 80/295 [00:00<00:01, 115.89it/s]
31%|###1 | 92/295 [00:00<00:01, 112.43it/s]
35%|###5 | 104/295 [00:00<00:01, 108.49it/s]
39%|###8 | 115/295 [00:01<00:01, 95.87it/s]
42%|####2 | 125/295 [00:01<00:01, 89.91it/s]
46%|####5 | 135/295 [00:01<00:01, 81.91it/s]
49%|####8 | 144/295 [00:01<00:02, 74.88it/s]
52%|#####1 | 152/295 [00:01<00:02, 69.85it/s]
54%|#####4 | 160/295 [00:01<00:02, 65.74it/s]
57%|#####6 | 167/295 [00:01<00:02, 62.68it/s]
59%|#####8 | 174/295 [00:02<00:02, 59.98it/s]
61%|######1 | 181/295 [00:02<00:01, 57.14it/s]
63%|######3 | 187/295 [00:02<00:01, 54.19it/s]
65%|######5 | 193/295 [00:02<00:01, 51.45it/s]
67%|######7 | 199/295 [00:02<00:01, 50.61it/s]
69%|######9 | 205/295 [00:02<00:01, 49.93it/s]
71%|#######1 | 210/295 [00:02<00:01, 49.34it/s]
73%|#######2 | 215/295 [00:02<00:01, 48.47it/s]
75%|#######4 | 220/295 [00:02<00:01, 47.78it/s]
76%|#######6 | 225/295 [00:03<00:01, 47.13it/s]
78%|#######7 | 230/295 [00:03<00:01, 46.44it/s]
80%|#######9 | 235/295 [00:03<00:01, 45.74it/s]
81%|########1 | 240/295 [00:03<00:01, 45.07it/s]
83%|########3 | 245/295 [00:03<00:01, 44.44it/s]
85%|########4 | 250/295 [00:03<00:01, 43.79it/s]
86%|########6 | 255/295 [00:03<00:00, 43.19it/s]
88%|########8 | 260/295 [00:03<00:00, 42.54it/s]
90%|########9 | 265/295 [00:04<00:00, 41.98it/s]
92%|#########1| 270/295 [00:04<00:00, 41.41it/s]
93%|#########3| 275/295 [00:04<00:00, 40.76it/s]
95%|#########4| 280/295 [00:04<00:00, 40.29it/s]
97%|#########6| 285/295 [00:04<00:00, 39.88it/s]
98%|#########7| 289/295 [00:04<00:00, 39.49it/s]
99%|#########9| 293/295 [00:04<00:00, 38.81it/s]
100%|##########| 295/295 [00:04<00:00, 61.46it/s]
Test Loss - 0.5892485745882584, AUC - 0.5474197292312352
0%| | 0/10 [00:00<?, ?it/s]
10%|# | 1/10 [00:00<00:01, 6.00it/s]
20%|## | 2/10 [00:00<00:01, 6.65it/s]
30%|### | 3/10 [00:00<00:00, 7.12it/s]
40%|#### | 4/10 [00:00<00:00, 7.43it/s]
50%|##### | 5/10 [00:00<00:00, 7.61it/s]
60%|###### | 6/10 [00:00<00:00, 7.73it/s]
70%|####### | 7/10 [00:00<00:00, 7.81it/s]
80%|######## | 8/10 [00:01<00:00, 7.87it/s]
90%|######### | 9/10 [00:01<00:00, 7.91it/s]
100%|##########| 10/10 [00:01<00:00, 7.93it/s]
100%|##########| 10/10 [00:01<00:00, 7.62it/s]
Total running time of the script: ( 6 minutes 2.650 seconds)
Estimated memory usage: 16980 MB