Multiprocess python

[pestipeti]

Runs in 3.3077sec, but I also train a CV model, so my machine is a bit busy.

import glob
import time
from multiprocessing import Pool

start = time.time()

files = glob.glob("./data/*.csv")
out = open("out.csv", "w")

def process(idx, src):

    with open(src, 'r') as in_file:

        if idx > 0:
            # Skip the header
            next(in_file)

        for line in in_file:
            out.write(line)


pool = Pool(processes=20)

for idx, f in enumerate(files):

    pool.apply_async(
        process,
        args=(idx, f)
    )

pool.close()
pool.join()
out.close()

end = time.time()
print(end - start)