bibi/src/filter-WOS.py

from pathlib import Path
import datetime
import pandas as pd


# 設定相對路徑
dbwebsite = 'WOS'
home_path = Path(__file__).parents[1]

checked_df = pd.read_excel(
    home_path.joinpath('SRDA_data/WOS比對檢查.xlsx')
)

# 去除尚未被檢查過的紀錄
checked_df.dropna(subset=['檢查結果'], inplace=True)

crawled_df = pd.read_csv(
    home_path.joinpath('crawled_data/export/ALL/WOS.csv')
)

df_keep = crawled_df[~crawled_df['UT'].isin(checked_df['UT'])]

print(f'剔除筆數：{crawled_df.shape[0] - df_keep.shape[0]}')


# 格式化输出日期
formatted_date = datetime.date.today().strftime('%Y%m%d')

export_path = home_path.joinpath('crawled_data/unchecked')
export_path.mkdir(parents=True, exist_ok=True)
df_keep.to_csv(
    export_path.joinpath(f'unchecked_{dbwebsite}_{formatted_date}.csv')
)