oshipka/util/process.py

import traceback
from time import time


def process_iterable(iterable: set, process_func, processed: set = None, errors: set = None,
                     f_args: list = None, f_kwargs: dict = None):
    processed = set() if not processed else processed
    errors = set() if not errors else errors
    f_args = list() if not f_args else f_args
    f_kwargs = dict() if not f_kwargs else f_kwargs

    to_process = iterable# - processed - errors
    tot_iterables = len(to_process)
    tot_start = time()

    for i, element in enumerate(to_process):
        print("Processing {}/{} -> {}".format(i + 1, tot_iterables, element))
        start = time()
        try:
            process_func(element, *f_args, **f_kwargs)
            processed.add(element)
        except Exception as e:
            errors.add(element)
            traceback.print_exc()
            print(errors)
            print(processed)
            print(e)

        end = time() - start
        tot_end = time() - tot_start
        avg_per_element = tot_end / (i + 1)
        eta = avg_per_element * (tot_iterables - i + 1)
        eta_min = eta / 60
        print("Processed. Time took: {:.0f}. Tot time: {:.0f}.".format(end, tot_end))
        print("Avg per element: {:.0f}. ETA: {:.0f} ({:.1f} min)".format(avg_per_element, eta, eta_min))