In [7]:
train_dataset = TrainDataset(train, transform=None)
for i in range(1):
image, label = train_dataset[i]
plt.imshow(image)
plt.title(f'label: {label}')
plt.show()
class CFG:
debug=False
apex=False
print_freq=20
num_workers=4
model_name="faster_rcnn_resnet50_fpn"
size=384
scheduler='CosineAnnealingWarmRestarts'
epochs=5
lr=1e-4
min_lr=1e-6
T_0=10 # CosineAnnealingWarmRestarts
batch_size=6
weight_decay=1e-6
gradient_accumulation_steps=1
max_grad_norm=1000
seed=42
num_classes=3
target_col=None
n_fold=5
trn_fold=[0, 1, 2, 3, 4]
train=True
inference=False
if CFG.debug:
CFG.epochs = 1
train = train.sample(n=1000, random_state=CFG.seed).reset_index(drop=True)
import sys
sys.path.append('../input/pytorch-image-models/pytorch-image-models-master')
import os
import math
import time
import random
import shutil
from pathlib import Path
from contextlib import contextmanager
from collections import defaultdict, Counter
import scipy as sp
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn.metrics import accuracy_score
from sklearn.model_selection import StratifiedKFold, KFold
from tqdm.auto import tqdm
from functools import partial
import cv2
from PIL import Image
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import Adam, SGD
import torchvision.models as models
from torch.nn.parameter import Parameter
from torch.utils.data import DataLoader, Dataset
from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau
from albumentations import (
Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip,
RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout,
IAAAdditiveGaussianNoise, Transpose
)
from albumentations.pytorch import ToTensorV2
from albumentations import ImageOnlyTransform
import timm
import warnings
warnings.filterwarnings('ignore')
if CFG.apex:
from apex import amp
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
TRAIN_PATH = '/kaggle/input/gods-3-0000/data/train'
TEST_PATH = '/kaggle/input/gods-3-0000/data/test'
import os
OUTPUT_DIR = './'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
train = pd.read_csv('/kaggle/input/rectified-data-gods/Train (6).csv')
test = pd.read_csv('/kaggle/input/rectified-data-gods/Test (3).csv')
target_cols = [column for column in train.columns if column !="filename"]
CFG.target_col = target_cols
from sklearn.metrics import log_loss
def get_score(y_true, y_pred):
num_classes = 6
total_log_loss = 0.0
for class_idx in range(num_classes):
class_true = y_true[:,class_idx]
class_pred = y_pred[:, class_idx]
class_log_loss = log_loss(class_true, class_pred)
total_log_loss += class_log_loss
mean_log_loss = total_log_loss / num_classes
return mean_log_loss
@contextmanager
def timer(name):
t0 = time.time()
LOGGER.info(f'[{name}] start')
yield
LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')
def init_logger(log_file=OUTPUT_DIR+'train.log'):
from logging import getLogger, INFO, FileHandler, Formatter, StreamHandler
logger = getLogger(__name__)
logger.setLevel(INFO)
handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=log_file)
handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
logger.addHandler(handler2)
return logger
LOGGER = init_logger()
def seed_torch(seed=42):
random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True
seed_torch(seed=CFG.seed)
--------------------------------------------------------------------------- NameError Traceback (most recent call last) Cell In[1], line 8 3 def get_score(y_true, y_pred): 5 return mean_log_loss ----> 8 @contextmanager 9 def timer(name): 10 t0 = time.time() 11 LOGGER.info(f'[{name}] start') NameError: name 'contextmanager' is not defined
class TrainDataset(Dataset):
def __init__(self, df, transform=None):
self.df = df
self.file_names = df['filename'].values
self.labels = df[target_cols].values
self.transform = transform
def __len__(self):
return len(self.df)
def __getitem__(self, idx):
file_name = self.file_names[idx]
file_path = f'{TRAIN_PATH}/{file_name}'
image = cv2.imread(file_path)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
if self.transform:
augmented = self.transform(image=image)
image = augmented['image']
label = torch.tensor(self.labels[idx]).long()
return image, label
class TestDataset(Dataset):
def __init__(self, df, transform=None):
self.df = df
self.file_names = df['filename'].values
self.transform = transform
def __len__(self):
return len(self.df)
def __getitem__(self, idx):
file_name = self.file_names[idx]
file_path = f'{TEST_PATH}/{file_name}'
image = cv2.imread(file_path)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
if self.transform:
augmented = self.transform(image=image)
image = augmented['image']
return image
train_dataset = TrainDataset(train, transform=None)
for i in range(1):
image, label = train_dataset[i]
plt.imshow(image)
plt.title(f'label: {label}')
plt.show()
import albumentations as A
def get_transforms(*, data):
if data == 'train':
return Compose([
#Resize(CFG.size, CFG.size),
#RandomResizedCrop(CFG.size, CFG.size),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
A.CoarseDropout(p=0.5),
Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
),
ToTensorV2(),
])
elif data == 'valid':
return Compose([
Resize(CFG.size, CFG.size),
Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
),
ToTensorV2(),
])
train_dataset = TrainDataset(train, transform=get_transforms(data='train'))
for i in range(1):
image, label = train_dataset[i]
plt.imshow(image[0])
plt.title(f'label: {label}')
plt.show()
folds = train.copy()
Fold = KFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)
for n, (train_index, val_index) in enumerate(Fold.split(folds, folds[CFG.target_col])):
folds.loc[val_index, 'fold'] = int(n)
folds['fold'] = folds['fold'].astype(int)
class CustomResNext(nn.Module):
def __init__(self, model_name='resnext50_32x4d', pretrained=False, num_classes = 10):
super().__init__()
self.model = timm.create_model(
CFG.model_name,
pretrained=pretrained,
drop_rate = 0.1,
drop_path_rate = 0.2,
num_classes = 10
)
def forward(self, x):
x = self.model(x)
return x
model = CustomResNext(model_name=CFG.model_name, pretrained=False)
train_dataset = TrainDataset(train, transform=get_transforms(data='train'))
train_loader = DataLoader(train_dataset, batch_size=4, shuffle=True,
num_workers=4, pin_memory=True, drop_last=True)
#make sure output is good
for image, label in train_loader:
output = model(image)
print(output.shape)
print(output)
break
torch.Size([4, 10])
tensor([[ 0.9627, 1.2202, -0.0071, 0.7995, 0.0166, -0.8795, 0.3679, -0.4606,
0.3227, -0.7633],
[-0.0212, -0.1883, 0.2700, -0.2932, 0.2946, 0.1969, -0.6448, 0.1243,
-0.8527, 0.8190],
[-0.6307, -0.5565, -0.0906, -0.4555, 0.0017, 0.4594, -0.6489, 0.3519,
-0.6807, 0.8727],
[-0.5950, -0.6208, 0.0567, -0.3117, 0.0603, 0.3581, -0.4585, 0.4179,
-0.9354, 0.8280]], grad_fn=<AddmmBackward0>)
class AverageMeter(object):
"""Computes and stores the average and current value"""
def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def asMinutes(s):
m = math.floor(s / 60)
s -= m * 60
return '%dm %ds' % (m, s)
def timeSince(since, percent):
now = time.time()
s = now - since
es = s / (percent)
rs = es - s
return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))
def train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
scores = AverageMeter()
# switch to train mode
model.train()
start = end = time.time()
global_step = 0
for step, (images, labels) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
images = images.to(device)
labels = labels.to(device)
batch_size = labels.size(0)
y_preds = model(images)
labels = labels.to(torch.float32)
y_preds = y_preds.to(torch.float32)
loss = criterion(y_preds, labels)
# record loss
losses.update(loss.item(), batch_size)
if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
if CFG.apex:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG.max_grad_norm)
if (step + 1) % CFG.gradient_accumulation_steps == 0:
optimizer.step()
optimizer.zero_grad()
global_step += 1
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if step % CFG.print_freq == 0 or step == (len(train_loader)-1):
print('Epoch: [{0}][{1}/{2}] '
'Data {data_time.val:.3f} ({data_time.avg:.3f}) '
'Elapsed {remain:s} '
'Loss: {loss.val:.4f}({loss.avg:.4f}) '
'Grad: {grad_norm:.4f} '
#'LR: {lr:.6f} '
.format(
epoch+1, step, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses,
remain=timeSince(start, float(step+1)/len(train_loader)),
grad_norm=grad_norm,
#lr=scheduler.get_lr()[0],
))
return losses.avg
def valid_fn(valid_loader, model, criterion, device):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
scores = AverageMeter()
# switch to evaluation mode
model.eval()
preds = []
start = end = time.time()
for step, (images, labels) in enumerate(valid_loader):
# measure data loading time
data_time.update(time.time() - end)
images = images.to(device)
labels = labels.to(device)
batch_size = labels.size(0)
# compute loss
with torch.no_grad():
y_preds = model(images)
labels = labels.to(torch.float32)
y_preds = y_preds.to(torch.float32)
loss = criterion(y_preds, labels)
losses.update(loss.item(), batch_size)
y_preds =torch.sigmoid(y_preds)
# record accuracy
y_preds = y_preds.to('cpu').numpy()
preds.append(y_preds)
if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if step % CFG.print_freq == 0 or step == (len(valid_loader)-1):
print('EVAL: [{0}/{1}] '
'Data {data_time.val:.3f} ({data_time.avg:.3f}) '
'Elapsed {remain:s} '
'Loss: {loss.val:.4f}({loss.avg:.4f}) '
.format(
step, len(valid_loader), batch_time=batch_time,
data_time=data_time, loss=losses,
remain=timeSince(start, float(step+1)/len(valid_loader)),
))
predictions = np.concatenate(preds)
return losses.avg, predictions
def inference(model, states, test_loader, device):
model.to(device)
tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i, (images) in tk0:
images = images.to(device)
avg_preds = []
for state in states:
model.load_state_dict(state['model'])
model.eval()
with torch.no_grad():
y_preds = model(images)
avg_preds.append(y_preds.to('cpu').numpy())
avg_preds = np.mean(avg_preds, axis=0)
probs.append(avg_preds)
probs = np.concatenate(probs)
return probs
def train_loop(folds, fold):
LOGGER.info(f"========== fold: {fold} training ==========")
# ====================================================
# loader
# ====================================================
trn_idx = folds[folds['fold'] != fold].index
val_idx = folds[folds['fold'] == fold].index
train_folds = folds.loc[trn_idx].reset_index(drop=True)
valid_folds = folds.loc[val_idx].reset_index(drop=True)
train_dataset = TrainDataset(train_folds,
transform=get_transforms(data='train'))
valid_dataset = TrainDataset(valid_folds,
transform=get_transforms(data='valid'))
train_loader = DataLoader(train_dataset,
batch_size=CFG.batch_size,
shuffle=True,
num_workers=CFG.num_workers, pin_memory=True, drop_last=True)
valid_loader = DataLoader(valid_dataset,
batch_size=CFG.batch_size,
shuffle=False,
num_workers=CFG.num_workers, pin_memory=True, drop_last=False)
# ====================================================
# scheduler
# ====================================================
def get_scheduler(optimizer):
if CFG.scheduler=='ReduceLROnPlateau':
scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG.factor, patience=CFG.patience, verbose=True, eps=CFG.eps)
elif CFG.scheduler=='CosineAnnealingLR':
scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)
elif CFG.scheduler=='CosineAnnealingWarmRestarts':
scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr, last_epoch=-1)
return scheduler
# ====================================================
# model & optimizer
# ====================================================
model = CustomResNext(CFG.model_name, pretrained=True)
model.to(device)
optimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)
scheduler = get_scheduler(optimizer)
# ====================================================
# apex
# ====================================================
if CFG.apex:
model, optimizer = amp.initialize(model, optimizer, opt_level='O1', verbosity=0)
# ====================================================
# loop
# ====================================================
criterion = nn.BCEWithLogitsLoss()
best_score = 50000
best_loss = np.inf
for epoch in range(CFG.epochs):
start_time = time.time()
# train
avg_loss = train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device)
# eval
avg_val_loss, preds = valid_fn(valid_loader, model, criterion, device)
valid_labels = valid_folds[CFG.target_col].values
if isinstance(scheduler, ReduceLROnPlateau):
scheduler.step(avg_val_loss)
elif isinstance(scheduler, CosineAnnealingLR):
scheduler.step()
elif isinstance(scheduler, CosineAnnealingWarmRestarts):
scheduler.step()
# scoring
score = get_score(valid_labels, preds)
elapsed = time.time() - start_time
LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f} avg_val_loss: {avg_val_loss:.4f} time: {elapsed:.0f}s')
LOGGER.info(f'Epoch {epoch+1} - Accuracy: {score}')
if score < best_score:
best_score = score
LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')
torch.save({'model': model.state_dict(),
'preds': preds},
OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth')
check_point = torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth')
#valid_folds[[str(c) for c in range(5)]] = check_point['preds']
#valid_folds['preds'] = check_point['preds'].argmax(1)
return
def main():
"""
Prepare: 1.train 2.test 3.submission 4.folds
"""
def get_result(result_df):
preds = result_df['preds'].values
labels = result_df[CFG.target_col].values
score = get_score(labels, preds)
LOGGER.info(f'Score: {score:<.5f}')
if CFG.train:
# train
oof_df = pd.DataFrame()
for fold in range(CFG.n_fold):
if fold in CFG.trn_fold:
train_loop(folds, fold)
#oof_df = pd.concat([oof_df, _oof_df])
#LOGGER.info(f"========== fold: {fold} result ==========")
#get_result(_oof_df)
# CV result
LOGGER.info(f"========== CV ==========")
#get_result(oof_df)
# save result
#oof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)
if CFG.inference:
# inference
model = CustomResNext(CFG.model_name, pretrained=False)
states = [torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth') for fold in CFG.trn_fold]
test_dataset = TestDataset(test, transform=get_transforms(data='valid'))
test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False,
num_workers=CFG.num_workers, pin_memory=True)
predictions = inference(model, states, test_loader, device)
# submission
test['label'] = predictions.argmax(1)
test[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)
if __name__ == '__main__':
main()
========== fold: 0 training ==========
model.safetensors: 0%| | 0.00/354M [00:00<?, ?B/s]
Epoch: [1][0/347] Data 0.601 (0.601) Elapsed 0m 7s (remain 40m 28s) Loss: 0.8524(0.8524) Grad: 6.5076 Epoch: [1][20/347] Data 0.000 (0.029) Elapsed 1m 50s (remain 28m 27s) Loss: 0.5450(0.6191) Grad: 3.9634 Epoch: [1][40/347] Data 0.000 (0.015) Elapsed 3m 33s (remain 26m 29s) Loss: 0.6062(0.5901) Grad: 4.3904 Epoch: [1][60/347] Data 0.000 (0.010) Elapsed 5m 16s (remain 24m 41s) Loss: 0.5035(0.5824) Grad: 4.0339 Epoch: [1][80/347] Data 0.000 (0.008) Elapsed 6m 59s (remain 22m 56s) Loss: 0.5603(0.5660) Grad: 3.2980
class TestDataset(Dataset):
def __init__(self, df, transform=None):
self.df = df
self.file_names = df['filename'].values
self.transform = transform
def __len__(self):
return len(self.df)
def __getitem__(self, idx):
file_name = self.file_names[idx]
file_path = f'{TEST_PATH}/{file_name}'
image = cv2.imread(file_path)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
if self.transform:
augmented = self.transform(image=image)
image = augmented['image']
return image
def get_transforms(*, data):
if data == 'train':
return Compose([
#Resize(CFG.size, CFG.size),
RandomResizedCrop(CFG.size, CFG.size),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
),
ToTensorV2(),
])
elif data == 'valid':
return Compose([
Resize(CFG.size, CFG.size),
Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
),
ToTensorV2(),
])
elif data == "hflip":
return Compose([
Resize(CFG.size, CFG.size),
HorizontalFlip(p=1.0),
Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
),
ToTensorV2(),
])
elif data == "vflip":
return Compose([
Resize(CFG.size, CFG.size),
VerticalFlip(p=1.0),
Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
),
ToTensorV2(),
])
CFG.model_name
model = CustomResNext(CFG.model_name, pretrained=False)
states = [torch.load(f"/kaggle/working/{CFG.model_name}_fold{fold}_best.pth") for fold in [0,1,2,3,4]]
tta_preds = []
for tta in ["valid", "hflip", "vflip"]:
tta_preds = []
test_dataset = TestDataset(test, transform=get_transforms(data=tta))
test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False,
num_workers=CFG.num_workers, pin_memory=True)
predictions = inference(model, states, test_loader, device)
tta_preds.append(predictions)
tta_preds = np.mean(tta_preds, axis=0)
def sigmoid(x):
return 1 / (1 + np.exp(-x))
predictions = sigmoid(predictions)
ss = pd.read_csv("/kaggle/input/rectified-data-gods/SampleSubmission (9).csv")
ss[CFG.target_col] = predictions
ss.to_csv("convnext_384_tta.csv", index=False)