Commit 048f0b1e authored by Oliver Wirth's avatar Oliver Wirth
Browse files

Add CNN models and training

parent 98ba33bf
......@@ -28,6 +28,13 @@
"sns.set_theme()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Load data"
]
},
{
"cell_type": "code",
"execution_count": 2,
......@@ -38,7 +45,7 @@
"data_path = Path('data')\n",
"train_path = data_path / 'train'\n",
"test_path = data_path / 'test'\n",
"model_path = Path('checkpoints')\n",
"runs_path = Path('runs')\n",
"\n",
"# Load labels\n",
"train_labels = pd.read_csv(f'{train_path}_label.csv')\n",
......@@ -91,7 +98,7 @@
"outputs": [],
"source": [
"# Preprocessing transformations\n",
"# Ideas: PCA, truncated SVD, MinMaxScaling\n",
"# TODO: PCA, truncated SVD, MinMaxScaling\n",
"transform = transforms.Compose([\n",
" StandardScaler().fit_transform,\n",
" torch.FloatTensor\n",
......@@ -104,7 +111,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
......@@ -115,22 +122,39 @@
"trainset, valset = random_split(trainset, [n_train, n_val])\n",
"\n",
"# Pack variable sized sequences for RNN\n",
"def collate(batch) :\n",
"def collate_pack(batch) :\n",
" samples, labels = zip(*batch)\n",
" samples = pack_sequence(samples, enforce_sorted = False)\n",
" labels = torch.tensor(labels)\n",
" return samples, labels\n",
"\n",
"# Crop sequences to same length for CNN\n",
"# TODO masking\n",
"def collate_crop(batch) :\n",
" samples, labels = zip(*batch)\n",
" length = min(x.size(0) for x in samples)\n",
" samples = torch.stack([x[-length:] for x in samples])\n",
" labels = torch.tensor(labels)\n",
" return samples, labels\n",
"\n",
"# Create data loader\n",
"batch_size = 8\n",
"trainloader = DataLoader(trainset, batch_size = batch_size, collate_fn = collate, shuffle = True)\n",
"testloader = DataLoader(testset , batch_size = batch_size, collate_fn = collate, shuffle = False)\n",
"valloader = DataLoader(valset , batch_size = batch_size, collate_fn = collate, shuffle = False)"
"collate_fn = collate_crop\n",
"trainloader = DataLoader(trainset, batch_size = batch_size, collate_fn = collate_fn, shuffle = True)\n",
"testloader = DataLoader(testset , batch_size = batch_size, collate_fn = collate_fn, shuffle = False)\n",
"valloader = DataLoader(valset , batch_size = batch_size, collate_fn = collate_fn, shuffle = False)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Models"
]
},
{
"cell_type": "code",
"execution_count": 165,
"execution_count": 39,
"metadata": {},
"outputs": [],
"source": [
......@@ -142,7 +166,7 @@
"dropout = .5\n",
"\n",
"# LSTM-based NN\n",
"class RNN(nn.Module) :\n",
"class LSTMRNN(nn.Module) :\n",
" \n",
" def __init__(self) :\n",
" super().__init__()\n",
......@@ -164,7 +188,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 40,
"metadata": {},
"outputs": [],
"source": [
......@@ -176,7 +200,7 @@
"dropout = .5\n",
"\n",
"# GRU-based NN\n",
"class RNN(nn.Module) :\n",
"class GRURNN(nn.Module) :\n",
" \n",
" def __init__(self) :\n",
" super().__init__()\n",
......@@ -198,62 +222,213 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 50,
"metadata": {},
"outputs": [],
"source": [
"# NN hyperparameters\n",
"input_size = 75\n",
"conv1_size = 64\n",
"conv1_kernel = 7\n",
"conv2_size = 64\n",
"conv2_kernel = 7\n",
"conv3_size = 64\n",
"conv3_kernel = 7\n",
"conv_stride = 2\n",
"pool_size = 16\n",
"hidden_size = 32\n",
"output_size = 2\n",
"dropout = .5\n",
"\n",
"# 1D convolutional NN\n",
"class CNN(nn.Module) :\n",
" \n",
" def __init__(self) :\n",
" super().__init__()\n",
" self.conv = nn.Sequential(\n",
" nn.Conv1d(input_size, conv1_size, kernel_size = conv1_kernel, stride = conv_stride),\n",
" nn.ReLU(),\n",
" nn.Dropout(dropout),\n",
" nn.Conv1d(conv1_size, conv2_size, kernel_size = conv2_kernel, stride = conv_stride),\n",
" nn.ReLU(),\n",
" nn.Dropout(dropout),\n",
" nn.Conv1d(conv2_size, conv3_size, kernel_size = conv3_kernel, stride = conv_stride),\n",
" nn.ReLU(),\n",
" nn.Dropout(dropout),\n",
" nn.AdaptiveMaxPool1d(pool_size)\n",
" )\n",
" self.fc = nn.Sequential(\n",
" nn.Linear(conv3_size * pool_size, hidden_size),\n",
" nn.ReLU(),\n",
" nn.Dropout(dropout),\n",
" nn.Linear(hidden_size, output_size)\n",
" )\n",
" \n",
" def forward(self, x) :\n",
" x = x.transpose(1, 2)\n",
" x = self.conv(x)\n",
" x = x.view(x.size(0), -1)\n",
" x = self.fc(x)\n",
" return x"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {},
"outputs": [],
"source": [
"# NN hyperparameters\n",
"input_size = 75\n",
"conv1_size = 64\n",
"conv1_kernel = 7\n",
"conv2_size = 64\n",
"conv2_kernel = 7\n",
"conv3_size = 64\n",
"conv3_kernel = 7\n",
"conv_stride = 2\n",
"pool_size = 16\n",
"hidden_size = 32\n",
"output_size = 2\n",
"dropout = .5\n",
"\n",
"# Separable convolution layer\n",
"class SepConv1d(nn.Module) :\n",
" \n",
" def __init__(self, in_channels, out_channels, kernel_size, stride = 1, padding = 0) :\n",
" super().__init__()\n",
" self.depthwise_conv = nn.Conv1d(in_channels, in_channels, kernel_size, stride, padding, groups = in_channels)\n",
" self.pointwise_conv = nn.Conv1d(in_channels, out_channels, kernel_size = 1)\n",
" \n",
" def forward(self, x) :\n",
" x = self.depthwise_conv(x)\n",
" x = self.pointwise_conv(x)\n",
" return x\n",
"\n",
"# CNN with separable 1D convolutions\n",
"class SepCNN(nn.Module) :\n",
" \n",
" def __init__(self) :\n",
" super().__init__()\n",
" self.conv = nn.Sequential(\n",
" SepConv1d(input_size, conv1_size, kernel_size = conv1_kernel, stride = conv_stride),\n",
" nn.ReLU(),\n",
" nn.Dropout(dropout),\n",
" SepConv1d(conv1_size, conv2_size, kernel_size = conv2_kernel, stride = conv_stride),\n",
" nn.ReLU(),\n",
" nn.Dropout(dropout),\n",
" SepConv1d(conv2_size, conv3_size, kernel_size = conv3_kernel, stride = conv_stride),\n",
" nn.ReLU(),\n",
" nn.Dropout(dropout),\n",
" nn.AdaptiveMaxPool1d(pool_size)\n",
" )\n",
" self.fc = nn.Sequential(\n",
" nn.Linear(conv3_size * pool_size, hidden_size),\n",
" nn.ReLU(),\n",
" nn.Dropout(dropout),\n",
" nn.Linear(hidden_size, output_size)\n",
" )\n",
" \n",
" def forward(self, x) :\n",
" x = x.transpose(1, 2)\n",
" x = self.conv(x)\n",
" x = x.view(x.size(0), -1)\n",
" x = self.fc(x)\n",
" return x"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Training parameters"
]
},
{
"cell_type": "code",
"execution_count": 104,
"metadata": {},
"outputs": [
{
"name": "stderr",
"name": "stdout",
"output_type": "stream",
"text": [
"/smartdata/uwwdp/.conda/envs/psda/lib/python3.7/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at /opt/conda/conda-bld/pytorch_1603729138878/work/c10/cuda/CUDAFunctions.cpp:100.)\n",
" return torch._C._cuda_getDeviceCount() > 0\n"
"SepCNN(\n",
" (conv): Sequential(\n",
" (0): SepConv1d(\n",
" (depthwise_conv): Conv1d(75, 75, kernel_size=(9,), stride=(3,), groups=75)\n",
" (pointwise_conv): Conv1d(75, 50, kernel_size=(1,), stride=(1,))\n",
" )\n",
" (1): ReLU()\n",
" (2): Dropout(p=0.5, inplace=False)\n",
" (3): SepConv1d(\n",
" (depthwise_conv): Conv1d(50, 50, kernel_size=(9,), stride=(3,), groups=50)\n",
" (pointwise_conv): Conv1d(50, 50, kernel_size=(1,), stride=(1,))\n",
" )\n",
" (4): ReLU()\n",
" (5): Dropout(p=0.5, inplace=False)\n",
" (6): SepConv1d(\n",
" (depthwise_conv): Conv1d(50, 50, kernel_size=(9,), stride=(3,), groups=50)\n",
" (pointwise_conv): Conv1d(50, 50, kernel_size=(1,), stride=(1,))\n",
" )\n",
" (7): ReLU()\n",
" (8): Dropout(p=0.5, inplace=False)\n",
" (9): AdaptiveMaxPool1d(output_size=16)\n",
" )\n",
" (fc): Sequential(\n",
" (0): Linear(in_features=800, out_features=32, bias=True)\n",
" (1): ReLU()\n",
" (2): Dropout(p=0.5, inplace=False)\n",
" (3): Linear(in_features=32, out_features=2, bias=True)\n",
" )\n",
")\n",
"The model 'SepCNN' has 36348 parameters\n"
]
},
{
"data": {
"text/plain": [
"RNN(\n",
" (rec): GRU(75, 32, batch_first=True)\n",
" (clf): Sequential(\n",
" (0): Dropout(p=0.5, inplace=False)\n",
" (1): Linear(in_features=32, out_features=16, bias=True)\n",
" (2): ReLU()\n",
" (3): Dropout(p=0.5, inplace=False)\n",
" (4): Linear(in_features=16, out_features=2, bias=True)\n",
" )\n",
")"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Learning rate and starting epoch\n",
"lr = 1e-3\n",
"epoch = 0\n",
"# Learning rate, starting epoch and max epochs\n",
"lr = 1e-3\n",
"epoch = 0\n",
"epochs = 100\n",
"\n",
"# Init model and optimizer\n",
"model = RNN()\n",
"model = SepCNN()\n",
"criterion = nn.CrossEntropyLoss()\n",
"optimizer = optim.Adam(model.parameters(), lr = lr)\n",
"\n",
"# LR scheduler\n",
"scheduler_warmup = 30\n",
"break_on_min_lr = True\n",
"min_lr = 1e-5\n",
"factor = .5\n",
"patience = 10\n",
"scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor = factor, patience = patience, verbose = True)\n",
"\n",
"# Loss/accuracy logs\n",
"loss_stats = pd.DataFrame(columns = ['train', 'val'])\n",
"acc_stats = pd.DataFrame(columns = ['train', 'val'])\n",
"\n",
"# CUDA\n",
"DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
"model.to(DEVICE)"
"model = model.to(DEVICE)\n",
"\n",
"# Model info\n",
"def prod(xs) :\n",
" return 1 if len(xs) == 0 else xs[0] * prod(xs[1:])\n",
"n_parameters = sum(prod(p.size()) for p in model.parameters())\n",
"print(model)\n",
"print(f'The model \\'{model.__class__.__name__}\\' has {n_parameters} parameters')"
]
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 89,
"metadata": {},
"outputs": [],
"source": [
"# Create run directory\n",
"run_path = model_path / str(datetime.datetime.now())\n",
"run_path = runs_path / str(datetime.datetime.now())\n",
"run_path.mkdir()\n",
"with open(run_path / 'model_architecture.txt', 'w') as f:\n",
" f.write(f'{model}\\n\\n{optimizer}\\nbatch_size: {batch_size}')"
......@@ -261,7 +436,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 90,
"metadata": {},
"outputs": [],
"source": [
......@@ -295,9 +470,16 @@
" return running_loss, running_correct"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train model"
]
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 91,
"metadata": {},
"outputs": [
{
......@@ -305,74 +487,288 @@
"output_type": "stream",
"text": [
"=== Epoch 1 ===\n",
" training loss = 0.6930, acc = 0.5359\n",
"validation loss = 0.6866, acc = 0.5531\n",
" training loss = 0.6933, acc = 0.5166\n",
"validation loss = 0.6907, acc = 0.5359\n",
"=== Epoch 2 ===\n",
" training loss = 0.6894, acc = 0.5204\n",
"validation loss = 0.6814, acc = 0.5796\n",
"=== Epoch 3 ===\n",
" training loss = 0.6792, acc = 0.5602\n",
"validation loss = 0.6753, acc = 0.5929\n",
"=== Epoch 4 ===\n",
" training loss = 0.6669, acc = 0.6011\n",
"validation loss = 0.6617, acc = 0.6239\n",
"=== Epoch 5 ===\n",
" training loss = 0.6411, acc = 0.6265\n",
"validation loss = 0.6496, acc = 0.6150\n",
"=== Epoch 6 ===\n",
" training loss = 0.6071, acc = 0.6862\n",
"validation loss = 0.6031, acc = 0.6726\n",
"=== Epoch 7 ===\n",
" training loss = 0.5705, acc = 0.7370\n",
"validation loss = 0.5814, acc = 0.7035\n",
"=== Epoch 8 ===\n",
" training loss = 0.4879, acc = 0.7812\n",
"validation loss = 0.5431, acc = 0.7434\n",
"=== Epoch 9 ===\n",
" training loss = 0.4156, acc = 0.8144\n",
"validation loss = 0.5644, acc = 0.7434\n",
"=== Epoch 10 ===\n",
" training loss = 0.3618, acc = 0.8320\n",
"validation loss = 0.5243, acc = 0.7743\n",
"=== Epoch 11 ===\n",
" training loss = 0.3511, acc = 0.8663\n",
"validation loss = 0.5075, acc = 0.7743\n",
"=== Epoch 12 ===\n",
" training loss = 0.2993, acc = 0.8851\n",
"validation loss = 0.5287, acc = 0.7743\n",
" training loss = 0.6918, acc = 0.5249\n",
"validation loss = 0.6911, acc = 0.5359\n",
"=== Epoch 13 ===\n",
" training loss = 0.2849, acc = 0.8972\n",
"validation loss = 0.5376, acc = 0.7920\n",
" training loss = 0.6919, acc = 0.5235\n",
"validation loss = 0.6909, acc = 0.5359\n",
"=== Epoch 14 ===\n",
" training loss = 0.2020, acc = 0.9304\n",
"validation loss = 0.6727, acc = 0.7478\n",
" training loss = 0.6912, acc = 0.5166\n",
"validation loss = 0.6923, acc = 0.5359\n",
"=== Epoch 15 ===\n",
" training loss = 0.2106, acc = 0.9238\n",
"validation loss = 0.6158, acc = 0.7743\n",
" training loss = 0.6953, acc = 0.5055\n",
"validation loss = 0.6929, acc = 0.5359\n",
"=== Epoch 16 ===\n",
" training loss = 0.2095, acc = 0.9293\n",
"validation loss = 0.5904, acc = 0.8186\n",
" training loss = 0.6912, acc = 0.5207\n",
"validation loss = 0.6935, acc = 0.5359\n",
"=== Epoch 17 ===\n",
" training loss = 0.1466, acc = 0.9602\n",
"validation loss = 0.5947, acc = 0.8097\n",
" training loss = 0.6888, acc = 0.5235\n",
"validation loss = 0.6970, acc = 0.5414\n",
"=== Epoch 18 ===\n",
" training loss = 0.1753, acc = 0.9536\n",
"validation loss = 0.6142, acc = 0.7876\n",
" training loss = 0.6929, acc = 0.5055\n",
"validation loss = 0.6964, acc = 0.5359\n",
"=== Epoch 19 ===\n",
" training loss = 0.1584, acc = 0.9392\n",
"validation loss = 0.6250, acc = 0.7920\n",
" training loss = 0.6873, acc = 0.5152\n",
"validation loss = 0.6969, acc = 0.5414\n",
"=== Epoch 20 ===\n",
" training loss = 0.3055, acc = 0.8862\n",
"validation loss = 0.5912, acc = 0.7345\n"
" training loss = 0.6818, acc = 0.5249\n",
"validation loss = 0.7036, acc = 0.5193\n",
"=== Epoch 21 ===\n",
" training loss = 0.6886, acc = 0.5110\n",
"validation loss = 0.7048, acc = 0.5304\n",
"=== Epoch 22 ===\n",
" training loss = 0.6884, acc = 0.5290\n",
"validation loss = 0.7018, acc = 0.5193\n",
"=== Epoch 23 ===\n",
" training loss = 0.6883, acc = 0.5428\n",
"validation loss = 0.6977, acc = 0.5414\n",
"=== Epoch 24 ===\n",
" training loss = 0.6837, acc = 0.5428\n",
"validation loss = 0.7006, acc = 0.5193\n",
"=== Epoch 25 ===\n",
" training loss = 0.6909, acc = 0.5428\n",
"validation loss = 0.6975, acc = 0.5359\n",
"=== Epoch 26 ===\n",
" training loss = 0.6853, acc = 0.5580\n",
"validation loss = 0.6990, acc = 0.4475\n",
"=== Epoch 27 ===\n",
" training loss = 0.6819, acc = 0.5622\n",
"validation loss = 0.7040, acc = 0.4033\n",
"=== Epoch 28 ===\n",
" training loss = 0.6833, acc = 0.5912\n",
"validation loss = 0.7016, acc = 0.5083\n",
"=== Epoch 29 ===\n",
" training loss = 0.6851, acc = 0.5773\n",
"validation loss = 0.6988, acc = 0.4475\n",
"=== Epoch 30 ===\n",
" training loss = 0.6748, acc = 0.6215\n",
"validation loss = 0.6995, acc = 0.5304\n",
"=== Epoch 31 ===\n",
" training loss = 0.6741, acc = 0.6367\n",
"validation loss = 0.7004, acc = 0.4420\n",
"=== Epoch 32 ===\n",
" training loss = 0.6668, acc = 0.6271\n",
"validation loss = 0.6939, acc = 0.5580\n",
"=== Epoch 33 ===\n",
" training loss = 0.6565, acc = 0.6119\n",
"validation loss = 0.6879, acc = 0.5359\n",
"=== Epoch 34 ===\n",
" training loss = 0.6428, acc = 0.6160\n",
"validation loss = 0.6832, acc = 0.5856\n",
"=== Epoch 35 ===\n",
" training loss = 0.6429, acc = 0.6326\n",
"validation loss = 0.6830, acc = 0.5083\n",
"=== Epoch 36 ===\n",
" training loss = 0.6471, acc = 0.6298\n",
"validation loss = 0.6603, acc = 0.5967\n",
"=== Epoch 37 ===\n",
" training loss = 0.6337, acc = 0.6354\n",
"validation loss = 0.6556, acc = 0.6188\n",
"=== Epoch 38 ===\n",
" training loss = 0.6192, acc = 0.6561\n",
"validation loss = 0.6393, acc = 0.6298\n",
"=== Epoch 39 ===\n",
" training loss = 0.6056, acc = 0.6685\n",
"validation loss = 0.6140, acc = 0.6685\n",
"=== Epoch 40 ===\n",
" training loss = 0.5844, acc = 0.6727\n",
"validation loss = 0.6421, acc = 0.6188\n",
"=== Epoch 41 ===\n",
" training loss = 0.5437, acc = 0.7431\n",
"validation loss = 0.5080, acc = 0.7790\n",
"=== Epoch 42 ===\n",
" training loss = 0.4812, acc = 0.7610\n",
"validation loss = 0.4835, acc = 0.7735\n",
"=== Epoch 43 ===\n",
" training loss = 0.5012, acc = 0.7693\n",
"validation loss = 0.5494, acc = 0.7238\n",
"=== Epoch 44 ===\n",
" training loss = 0.4842, acc = 0.7749\n",
"validation loss = 0.4775, acc = 0.7735\n",
"=== Epoch 45 ===\n",
" training loss = 0.4923, acc = 0.7762\n",
"validation loss = 0.5127, acc = 0.7845\n",
"=== Epoch 46 ===\n",
" training loss = 0.4837, acc = 0.7749\n",
"validation loss = 0.4842, acc = 0.7790\n",
"=== Epoch 47 ===\n",
" training loss = 0.4372, acc = 0.8177\n",
"validation loss = 0.4900, acc = 0.7624\n",
"=== Epoch 48 ===\n",
" training loss = 0.4591, acc = 0.7901\n",
"validation loss = 0.4810, acc = 0.7735\n",
"=== Epoch 49 ===\n",
" training loss = 0.4566, acc = 0.7845\n",
"validation loss = 0.4705, acc = 0.7790\n",
"=== Epoch 50 ===\n",
" training loss = 0.4343, acc = 0.8025\n",
"validation loss = 0.4492, acc = 0.7956\n",
"=== Epoch 51 ===\n",
" training loss = 0.4091, acc = 0.8273\n",
"validation loss = 0.4588, acc = 0.7624\n",
"=== Epoch 52 ===\n",
" training loss = 0.4339, acc = 0.8080\n",
"validation loss = 0.4376, acc = 0.8066\n",
"=== Epoch 53 ===\n",
" training loss = 0.4125, acc = 0.8204\n",
"validation loss = 0.4441, acc = 0.7845\n",
"=== Epoch 54 ===\n",
" training loss = 0.4001, acc = 0.8301\n",
"validation loss = 0.4705, acc = 0.7735\n",
"=== Epoch 55 ===\n",
" training loss = 0.4195, acc = 0.8149\n",
"validation loss = 0.4212, acc = 0.8066\n",
"=== Epoch 56 ===\n",
" training loss = 0.3917, acc = 0.8329\n",
"validation loss = 0.4247, acc = 0.8122\n",
"=== Epoch 57 ===\n",
" training loss = 0.4037, acc = 0.8315\n",
"validation loss = 0.4727, acc = 0.7680\n",
"=== Epoch 58 ===\n",
" training loss = 0.4145, acc = 0.8204\n",
"validation loss = 0.4312, acc = 0.8177\n",
"=== Epoch 59 ===\n",
" training loss = 0.3819, acc = 0.8246\n",
"validation loss = 0.4477, acc = 0.7790\n",
"=== Epoch 60 ===\n",
" training loss = 0.3778, acc = 0.8425\n",
"validation loss = 0.4317, acc = 0.7956\n",
"=== Epoch 61 ===\n",
" training loss = 0.3555, acc = 0.8564\n",
"validation loss = 0.5226, acc = 0.7293\n",
"=== Epoch 62 ===\n",
" training loss = 0.3846, acc = 0.8370\n",
"validation loss = 0.4231, acc = 0.7956\n",
"=== Epoch 63 ===\n",
" training loss = 0.3698, acc = 0.8522\n",
"validation loss = 0.4293, acc = 0.7956\n",
"=== Epoch 64 ===\n",
" training loss = 0.3363, acc = 0.8550\n",
"validation loss = 0.3876, acc = 0.8398\n",
"=== Epoch 65 ===\n",
" training loss = 0.3448, acc = 0.8619\n",
"validation loss = 0.4254, acc = 0.8011\n",
"=== Epoch 66 ===\n",
" training loss = 0.3274, acc = 0.8729\n",
"validation loss = 0.4511, acc = 0.7956\n",
"=== Epoch 67 ===\n",
" training loss = 0.3803, acc = 0.8439\n",
"validation loss = 0.3979, acc = 0.8343\n",
"=== Epoch 68 ===\n",
" training loss = 0.3393, acc = 0.8646\n",
"validation loss = 0.3803, acc = 0.8508\n",
"=== Epoch 69 ===\n",
" training loss = 0.3203, acc = 0.8688\n",
"validation loss = 0.3662, acc = 0.8508\n",
"=== Epoch 70 ===\n",
" training loss = 0.3396, acc = 0.8646\n",
"validation loss = 0.4029, acc = 0.8122\n",
"=== Epoch 71 ===\n",
" training loss = 0.3121, acc = 0.8771\n",
"validation loss = 0.3760, acc = 0.8287\n",
"=== Epoch 72 ===\n",
" training loss = 0.2864, acc = 0.8826\n",
"validation loss = 0.3655, acc = 0.8398\n",
"=== Epoch 73 ===\n",
" training loss = 0.3109, acc = 0.8743\n",
"validation loss = 0.3651, acc = 0.8398\n",
"=== Epoch 74 ===\n",
" training loss = 0.3039, acc = 0.8729\n",
"validation loss = 0.3676, acc = 0.8343\n",
"=== Epoch 75 ===\n",
" training loss = 0.3033, acc = 0.8812\n",
"validation loss = 0.3618, acc = 0.8287\n",
"=== Epoch 76 ===\n",
" training loss = 0.3041, acc = 0.8826\n",
"validation loss = 0.3573, acc = 0.8398\n",
"=== Epoch 77 ===\n",
" training loss = 0.2651, acc = 0.8923\n",
"validation loss = 0.4185, acc = 0.8122\n",
"=== Epoch 78 ===\n",
" training loss = 0.3009, acc = 0.8771\n",
"validation loss = 0.4588, acc = 0.7845\n",
"=== Epoch 79 ===\n",
" training loss = 0.2981, acc = 0.8798\n",
"validation loss = 0.3645, acc = 0.8508\n",
"=== Epoch 80 ===\n",
" training loss = 0.2955, acc = 0.8936\n",
"validation loss = 0.4123, acc = 0.8343\n",
"=== Epoch 81 ===\n",
" training loss = 0.3238, acc = 0.8715\n",
"validation loss = 0.3819, acc = 0.8453\n",
"=== Epoch 82 ===\n",
" training loss = 0.2753, acc = 0.8798\n",
"validation loss = 0.3578, acc = 0.8564\n",
"=== Epoch 83 ===\n",
" training loss = 0.2764, acc = 0.8895\n",
"validation loss = 0.3555, acc = 0.8508\n",
"=== Epoch 84 ===\n",
" training loss = 0.2825, acc = 0.8826\n",
"validation loss = 0.3627, acc = 0.8619\n",
"=== Epoch 85 ===\n",
" training loss = 0.2461, acc = 0.8923\n",
"validation loss = 0.3926, acc = 0.8398\n",
"=== Epoch 86 ===\n",
" training loss = 0.2709, acc = 0.8936\n",
"validation loss = 0.3582, acc = 0.8398\n",
"=== Epoch 87 ===\n",
" training loss = 0.2935, acc = 0.8826\n",
"validation loss = 0.3618, acc = 0.8398\n",
"=== Epoch 88 ===\n",
" training loss = 0.2542, acc = 0.8867\n",
"validation loss = 0.3287, acc = 0.8619\n",