36 lines
1.3 KiB
Python
36 lines
1.3 KiB
Python
import traceback
|
|
from time import time
|
|
|
|
|
|
def process_iterable(iterable: set, process_func, processed: set = None, errors: set = None,
|
|
f_args: list = None, f_kwargs: dict = None):
|
|
processed = set() if not processed else processed
|
|
errors = set() if not errors else errors
|
|
f_args = list() if not f_args else f_args
|
|
f_kwargs = dict() if not f_kwargs else f_kwargs
|
|
|
|
to_process = iterable# - processed - errors
|
|
tot_iterables = len(to_process)
|
|
tot_start = time()
|
|
|
|
for i, element in enumerate(to_process):
|
|
print("Processing {}/{} -> {}".format(i + 1, tot_iterables, element))
|
|
start = time()
|
|
try:
|
|
process_func(element, *f_args, **f_kwargs)
|
|
processed.add(element)
|
|
except Exception as e:
|
|
errors.add(element)
|
|
traceback.print_exc()
|
|
print(errors)
|
|
print(processed)
|
|
print(e)
|
|
|
|
end = time() - start
|
|
tot_end = time() - tot_start
|
|
avg_per_element = tot_end / (i + 1)
|
|
eta = avg_per_element * (tot_iterables - i + 1)
|
|
eta_min = eta / 60
|
|
print("Processed. Time took: {:.0f}. Tot time: {:.0f}.".format(end, tot_end))
|
|
print("Avg per element: {:.0f}. ETA: {:.0f} ({:.1f} min)".format(avg_per_element, eta, eta_min))
|