concat_csv.py

#!/usr/bin/python

import argparse

import pandas as pd

parser = argparse.ArgumentParser(
    description="Small utility to safely concatenate multiple csv files, "
                "especially useful here when concatenating multiple `results.csv` files.",
    epilog="""Usage example:
find ./results -name 'results.csv' -exec python concat_csv.py results.csv {} +
""",
    formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('target', type=str, help='The target file that will contain the concatenated csv files.')
parser.add_argument('sources',  nargs='*', help='the csv files to be concatenated.')
args = parser.parse_args()

print(f"concatenating following files to `{args.target}`:\n{args.sources}")
(pd.concat([pd.read_csv(file) for file in args.sources], ignore_index=True)
   .drop_duplicates()
   .to_csv(args.target, index=False))