process

oshipka/persistance/__init__.py

@@ -104,14 +104,14 @@ class ModelController(ModelJsonEncoder):
         def handle_withs(data, with_, depth, withs_used):
             if isinstance(data, InstrumentedList):
                 if depth >= MAX_DEPTH:
-                    return [e.serialize_flat() for e in data]
+                    return [e.serialize() for e in data]
                 else:
-                    return [e.serialize_flat(with_=with_, depth=depth + 1, withs_used=withs_used) for e in data]
+                    return [e.serialize(with_=with_, depth=depth + 1, withs_used=withs_used) for e in data]
             else:
                 if depth >= MAX_DEPTH:
-                    return data.serialize_flat()
+                    return data.serialize()
                 else:
-                    return data.serialize_flat(with_=with_, depth=depth + 1, withs_used=withs_used)
+                    return data.serialize(with_=with_, depth=depth + 1, withs_used=withs_used)
 
         if not with_:
             with_ = []
@@ -144,7 +144,7 @@ class ModelController(ModelJsonEncoder):
                     if isinstance(data, InstrumentedList):
                         continue  # pragma: no cover
                     fields[field] = data
-                except TypeError:
+                except TypeError as e:
                     pass  # Don't assign anything
             # a json-encodable dict
             return fields
@@ -195,6 +195,8 @@ def register_filters(app):
         :param dt a datetime object
         :return:
         """
+        if not dt:
+            return dt
         if type(dt) is str:
             dt = solve_query(dt)
         return dt.strftime(formatting)

oshipka/util/process.py

@@ -0,0 +1,35 @@
+import traceback
+from time import time
+
+
+def process_iterable(iterable: set, process_func, processed: set = None, errors: set = None,
+                     f_args: list = None, f_kwargs: dict = None):
+    processed = set() if not processed else processed
+    errors = set() if not errors else errors
+    f_args = list() if not f_args else f_args
+    f_kwargs = dict() if not f_kwargs else f_kwargs
+
+    to_process = iterable - processed - errors
+    tot_iterables = len(to_process)
+    tot_start = time()
+
+    for i, element in enumerate(to_process):
+        print("Processing {}/{} -> {}".format(i + 1, tot_iterables, element))
+        start = time()
+        try:
+            process_func(element, *f_args, **f_kwargs)
+            processed.add(element)
+        except Exception as e:
+            errors.add(element)
+            traceback.print_exc()
+            print(errors)
+            print(processed)
+            print(e)
+
+        end = time() - start
+        tot_end = time() - tot_start
+        avg_per_element = tot_end / (i + 1)
+        eta = avg_per_element * (tot_iterables - i + 1)
+        eta_min = eta / 60
+        print("Processed. Time took: {:.0f}. Tot time: {:.0f}.".format(end, tot_end))
+        print("Avg per element: {:.0f}. ETA: {:.0f} ({:.1f} min)".format(avg_per_element, eta, eta_min))