From 91f288997c90d7b9c8a319ef0943c233023af295 Mon Sep 17 00:00:00 2001
From: Daniel Tsvetkov <danieltcv@gmail.com>
Date: Tue, 5 May 2020 15:50:35 +0200
Subject: [PATCH] auto process csv files

---
 bootstrap/config.py                  |  3 ++-
 bootstrap/data_static/_process_order |  0
 bootstrap/run.py                     |  2 ++
 oshipka/persistance/__init__.py      | 34 ++++++++++++++++++++++++++--
 4 files changed, 36 insertions(+), 3 deletions(-)
 create mode 100644 bootstrap/data_static/_process_order

diff --git a/bootstrap/config.py b/bootstrap/config.py
index c2f6fb7..7c80208 100644
--- a/bootstrap/config.py
+++ b/bootstrap/config.py
@@ -3,6 +3,7 @@ import os
 basepath = os.path.dirname(os.path.realpath(__file__))
 
 DATA_DIR = os.path.join(basepath, "data")
+STATIC_DATA_DIR = os.path.join(basepath, "data_static")
 
 TASKS_DIR = os.path.join(DATA_DIR, "tasks")
 TASKS_IN_DIR = os.path.join(TASKS_DIR, "in")
@@ -16,5 +17,5 @@ TEMPLATES_FOLDER = os.path.join(basepath, "webapp", "templates")
 STATIC_FOLDER = os.path.join(basepath, "webapp", "static")
 
 MAKEDIRS = [
-    DATA_DIR, TASKS_DIR, TASKS_IN_DIR, TASKS_PROC_DIR, TASKS_BUF_DIR,
+    DATA_DIR, STATIC_DATA_DIR, TASKS_DIR, TASKS_IN_DIR, TASKS_PROC_DIR, TASKS_BUF_DIR,
 ]
diff --git a/bootstrap/data_static/_process_order b/bootstrap/data_static/_process_order
new file mode 100644
index 0000000..e69de29
diff --git a/bootstrap/run.py b/bootstrap/run.py
index 44799f3..7254975 100644
--- a/bootstrap/run.py
+++ b/bootstrap/run.py
@@ -1,10 +1,12 @@
 from config import TEMPLATES_FOLDER, STATIC_FOLDER
 from oshipka import init_db
+from oshipka.persistance import populate_static
 
 from populate import populate_db
 from webapp.app import app
 
 if init_db(app):
+    populate_static(app)
     populate_db(app)
 
 app.template_folder = TEMPLATES_FOLDER
diff --git a/oshipka/persistance/__init__.py b/oshipka/persistance/__init__.py
index 9bf0fcb..9bc54d8 100644
--- a/oshipka/persistance/__init__.py
+++ b/oshipka/persistance/__init__.py
@@ -1,11 +1,13 @@
+import csv
 import datetime
 import json
 import os
 import re
+from importlib import import_module
 from json import JSONEncoder
 from uuid import uuid4
 
-from config import SQLALCHEMY_DATABASE_URI, MAKEDIRS, DATABASE_FILE, SEARCH_INDEX_PATH
+from config import SQLALCHEMY_DATABASE_URI, MAKEDIRS, DATABASE_FILE, SEARCH_INDEX_PATH, STATIC_DATA_DIR
 from flask_security import RoleMixin, UserMixin
 from flask_security import Security, SQLAlchemyUserDatastore
 from flask_sqlalchemy import SQLAlchemy
@@ -48,7 +50,9 @@ class LiberalBoolean(TypeDecorator):
 
     def process_bind_param(self, value, dialect):
         if value is not None:
-            value = bool(int(value))
+            if hasattr(value, 'isdigit') and value.isdigit():
+                value = int(value)
+            value = bool(value)
         return value
 
 
@@ -247,3 +251,29 @@ def init_db(app):
     global index_service
     index_service.proxied = IndexService(config=app.config, session=db.session)
     return rv
+
+
+def populate_static(app):
+    with app.app_context():
+        models = import_module("webapp.models")
+        model_names = set([f.split('.csv')[0] for f in os.listdir(STATIC_DATA_DIR) if f.endswith(".csv")])
+        process_order_file = os.path.join(STATIC_DATA_DIR, "_process_order")
+        ordered_model_names = []
+        # process first ordered if exists
+        if os.path.exists(process_order_file):
+            with open(process_order_file) as f:
+                for line in f.readlines():
+                    line = line.strip()
+                    if line:
+                        ordered_model_names.append(line)
+                        model_names.remove(line)
+        for model_name in model_names:
+            ordered_model_names.append(model_name)
+        for model_name in ordered_model_names:
+            model = getattr(models, model_name)
+            with open(os.path.join(STATIC_DATA_DIR, "{}.csv".format(model_name))) as f:
+                reader = csv.DictReader(f)
+                for row in reader:
+                    instance = model(**row)
+                    db.session.add(instance)
+            db.session.commit()