From 76f12c61f73cf64202fb3f27fa93d44d80ba98fb Mon Sep 17 00:00:00 2001
From: Daniel Tsvetkov <danieltcv@gmail.com>
Date: Mon, 18 May 2020 17:57:54 +0200
Subject: [PATCH] process

---
 oshipka/persistance/__init__.py | 12 ++++++-----
 oshipka/util/process.py         | 35 +++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 5 deletions(-)
 create mode 100644 oshipka/util/process.py

diff --git a/oshipka/persistance/__init__.py b/oshipka/persistance/__init__.py
index 9bc54d8..77a099f 100644
--- a/oshipka/persistance/__init__.py
+++ b/oshipka/persistance/__init__.py
@@ -104,14 +104,14 @@ class ModelController(ModelJsonEncoder):
         def handle_withs(data, with_, depth, withs_used):
             if isinstance(data, InstrumentedList):
                 if depth >= MAX_DEPTH:
-                    return [e.serialize_flat() for e in data]
+                    return [e.serialize() for e in data]
                 else:
-                    return [e.serialize_flat(with_=with_, depth=depth + 1, withs_used=withs_used) for e in data]
+                    return [e.serialize(with_=with_, depth=depth + 1, withs_used=withs_used) for e in data]
             else:
                 if depth >= MAX_DEPTH:
-                    return data.serialize_flat()
+                    return data.serialize()
                 else:
-                    return data.serialize_flat(with_=with_, depth=depth + 1, withs_used=withs_used)
+                    return data.serialize(with_=with_, depth=depth + 1, withs_used=withs_used)
 
         if not with_:
             with_ = []
@@ -144,7 +144,7 @@ class ModelController(ModelJsonEncoder):
                     if isinstance(data, InstrumentedList):
                         continue  # pragma: no cover
                     fields[field] = data
-                except TypeError:
+                except TypeError as e:
                     pass  # Don't assign anything
             # a json-encodable dict
             return fields
@@ -195,6 +195,8 @@ def register_filters(app):
         :param dt a datetime object
         :return:
         """
+        if not dt:
+            return dt
         if type(dt) is str:
             dt = solve_query(dt)
         return dt.strftime(formatting)
diff --git a/oshipka/util/process.py b/oshipka/util/process.py
new file mode 100644
index 0000000..e44cf20
--- /dev/null
+++ b/oshipka/util/process.py
@@ -0,0 +1,35 @@
+import traceback
+from time import time
+
+
+def process_iterable(iterable: set, process_func, processed: set = None, errors: set = None,
+                     f_args: list = None, f_kwargs: dict = None):
+    processed = set() if not processed else processed
+    errors = set() if not errors else errors
+    f_args = list() if not f_args else f_args
+    f_kwargs = dict() if not f_kwargs else f_kwargs
+
+    to_process = iterable - processed - errors
+    tot_iterables = len(to_process)
+    tot_start = time()
+
+    for i, element in enumerate(to_process):
+        print("Processing {}/{} -> {}".format(i + 1, tot_iterables, element))
+        start = time()
+        try:
+            process_func(element, *f_args, **f_kwargs)
+            processed.add(element)
+        except Exception as e:
+            errors.add(element)
+            traceback.print_exc()
+            print(errors)
+            print(processed)
+            print(e)
+
+        end = time() - start
+        tot_end = time() - tot_start
+        avg_per_element = tot_end / (i + 1)
+        eta = avg_per_element * (tot_iterables - i + 1)
+        eta_min = eta / 60
+        print("Processed. Time took: {:.0f}. Tot time: {:.0f}.".format(end, tot_end))
+        print("Avg per element: {:.0f}. ETA: {:.0f} ({:.1f} min)".format(avg_per_element, eta, eta_min))