CheXNet/src/reduce_dataset.py

#!/usr/bin/env python
# coding: utf-8

from omegaconf import OmegaConf
from glob import glob
import pandas as pd
import os

GENERAL_CONST_PATH = os.path.join('src', 'const.yaml')

def subsample(data_dir, data_file, data_image_dir):
    df = pd.read_csv(os.path.join(data_dir, data_file))
    df = df.sample(n=df.shape[0] // 45)

    data_image_paths = {os.path.basename(x): x for x in glob(os.path.join(data_dir, *data_image_dir))}
    [data_image_paths.pop(img) for img in list(df['Image Index'])]
    [os.remove(img) for img in data_image_paths.values()]

if __name__ == '__main__':
    general_const = OmegaConf.load(os.path.join(os.getcwd(), GENERAL_CONST_PATH))
    subsample(data_dir=os.path.join(os.getcwd(), *general_const.DATA_DIR),
              data_file=general_const.DATA_FILE,
              data_image_dir=general_const.IMAGE_PATH)