diff --git a/oshipka/persistance/__init__.py b/oshipka/persistance/__init__.py index 9bc54d8..77a099f 100644 --- a/oshipka/persistance/__init__.py +++ b/oshipka/persistance/__init__.py @@ -104,14 +104,14 @@ class ModelController(ModelJsonEncoder): def handle_withs(data, with_, depth, withs_used): if isinstance(data, InstrumentedList): if depth >= MAX_DEPTH: - return [e.serialize_flat() for e in data] + return [e.serialize() for e in data] else: - return [e.serialize_flat(with_=with_, depth=depth + 1, withs_used=withs_used) for e in data] + return [e.serialize(with_=with_, depth=depth + 1, withs_used=withs_used) for e in data] else: if depth >= MAX_DEPTH: - return data.serialize_flat() + return data.serialize() else: - return data.serialize_flat(with_=with_, depth=depth + 1, withs_used=withs_used) + return data.serialize(with_=with_, depth=depth + 1, withs_used=withs_used) if not with_: with_ = [] @@ -144,7 +144,7 @@ class ModelController(ModelJsonEncoder): if isinstance(data, InstrumentedList): continue # pragma: no cover fields[field] = data - except TypeError: + except TypeError as e: pass # Don't assign anything # a json-encodable dict return fields @@ -195,6 +195,8 @@ def register_filters(app): :param dt a datetime object :return: """ + if not dt: + return dt if type(dt) is str: dt = solve_query(dt) return dt.strftime(formatting) diff --git a/oshipka/util/process.py b/oshipka/util/process.py new file mode 100644 index 0000000..e44cf20 --- /dev/null +++ b/oshipka/util/process.py @@ -0,0 +1,35 @@ +import traceback +from time import time + + +def process_iterable(iterable: set, process_func, processed: set = None, errors: set = None, + f_args: list = None, f_kwargs: dict = None): + processed = set() if not processed else processed + errors = set() if not errors else errors + f_args = list() if not f_args else f_args + f_kwargs = dict() if not f_kwargs else f_kwargs + + to_process = iterable - processed - errors + tot_iterables = len(to_process) + tot_start = time() + + for i, element in enumerate(to_process): + print("Processing {}/{} -> {}".format(i + 1, tot_iterables, element)) + start = time() + try: + process_func(element, *f_args, **f_kwargs) + processed.add(element) + except Exception as e: + errors.add(element) + traceback.print_exc() + print(errors) + print(processed) + print(e) + + end = time() - start + tot_end = time() - tot_start + avg_per_element = tot_end / (i + 1) + eta = avg_per_element * (tot_iterables - i + 1) + eta_min = eta / 60 + print("Processed. Time took: {:.0f}. Tot time: {:.0f}.".format(end, tot_end)) + print("Avg per element: {:.0f}. ETA: {:.0f} ({:.1f} min)".format(avg_per_element, eta, eta_min))