import traceback from time import time def process_iterable(iterable: set, process_func, processed: set = None, errors: set = None, f_args: list = None, f_kwargs: dict = None): processed = set() if not processed else processed errors = set() if not errors else errors f_args = list() if not f_args else f_args f_kwargs = dict() if not f_kwargs else f_kwargs to_process = iterable# - processed - errors tot_iterables = len(to_process) tot_start = time() for i, element in enumerate(to_process): print("Processing {}/{} -> {}".format(i + 1, tot_iterables, element)) start = time() try: process_func(element, *f_args, **f_kwargs) processed.add(element) except Exception as e: errors.add(element) traceback.print_exc() print(errors) print(processed) print(e) end = time() - start tot_end = time() - tot_start avg_per_element = tot_end / (i + 1) eta = avg_per_element * (tot_iterables - i + 1) eta_min = eta / 60 print("Processed. Time took: {:.0f}. Tot time: {:.0f}.".format(end, tot_end)) print("Avg per element: {:.0f}. ETA: {:.0f} ({:.1f} min)".format(avg_per_element, eta, eta_min))