bibi/src/merge_crawled_data-ncl.py

from pathlib import Path
import pandas as pd
import csv

# 設定欄位長度上限為100萬個字元
csv.field_size_limit(1000000)


def merge_txt_files_to_csv(input_folder_path, output_folder_path):
    output_folder = Path(output_folder_path)
    output_folder.mkdir(parents=True, exist_ok=True)
    for folder_path in Path(input_folder_path).iterdir():
        print(f'folder_path is {folder_path}')
        if folder_path.is_dir():
            csv_file_name = folder_path.name + ".csv"
            csv_file_path = output_folder / csv_file_name
            dfs = []
            for f in folder_path.glob("*.csv"):
                try:
                    # df = pd.read_csv(f, sep=',', engine='python',
                    #                  on_bad_lines='warn'  # 先忽略格式問題
                    #                  )
                    df = pd.read_csv(f, header=0, sep=',',
                                     quotechar='"',
                                     skipinitialspace=True,
                                     #  quoting=csv.QUOTE_NONE,
                                     encoding='utf-8', engine='python',
                                     dtype=str,
                                     on_bad_lines='warn'  # 先忽略格式問題
                                     )
                    dfs.append(df)
                except ValueError as e:
                    print(f"Error reading file {f}: {e}")
            if len(dfs) > 0:
                combined_df = pd.concat(dfs, ignore_index=True)
                combined_df.to_csv(csv_file_path, index=False)

    print("Done!")


this_file_path = Path(__file__)
input_folder_path = this_file_path.parents[1].joinpath(f"crawler_data")

output_folder_path = this_file_path.parents[1].joinpath(
    f"crawler_data/out/NCL")
merge_txt_files_to_csv(input_folder_path, output_folder_path)