auto process csv files

bootstrap/config.py

@@ -3,6 +3,7 @@ import os
 basepath = os.path.dirname(os.path.realpath(__file__))
 
 DATA_DIR = os.path.join(basepath, "data")
+STATIC_DATA_DIR = os.path.join(basepath, "data_static")
 
 TASKS_DIR = os.path.join(DATA_DIR, "tasks")
 TASKS_IN_DIR = os.path.join(TASKS_DIR, "in")
@@ -16,5 +17,5 @@ TEMPLATES_FOLDER = os.path.join(basepath, "webapp", "templates")
 STATIC_FOLDER = os.path.join(basepath, "webapp", "static")
 
 MAKEDIRS = [
-    DATA_DIR, TASKS_DIR, TASKS_IN_DIR, TASKS_PROC_DIR, TASKS_BUF_DIR,
+    DATA_DIR, STATIC_DATA_DIR, TASKS_DIR, TASKS_IN_DIR, TASKS_PROC_DIR, TASKS_BUF_DIR,
 ]

bootstrap/run.py

@@ -1,10 +1,12 @@
 from config import TEMPLATES_FOLDER, STATIC_FOLDER
 from oshipka import init_db
+from oshipka.persistance import populate_static
 
 from populate import populate_db
 from webapp.app import app
 
 if init_db(app):
+    populate_static(app)
     populate_db(app)
 
 app.template_folder = TEMPLATES_FOLDER

oshipka/persistance/__init__.py

@@ -1,11 +1,13 @@
+import csv
 import datetime
 import json
 import os
 import re
+from importlib import import_module
 from json import JSONEncoder
 from uuid import uuid4
 
-from config import SQLALCHEMY_DATABASE_URI, MAKEDIRS, DATABASE_FILE, SEARCH_INDEX_PATH
+from config import SQLALCHEMY_DATABASE_URI, MAKEDIRS, DATABASE_FILE, SEARCH_INDEX_PATH, STATIC_DATA_DIR
 from flask_security import RoleMixin, UserMixin
 from flask_security import Security, SQLAlchemyUserDatastore
 from flask_sqlalchemy import SQLAlchemy
@@ -48,7 +50,9 @@ class LiberalBoolean(TypeDecorator):
 
     def process_bind_param(self, value, dialect):
         if value is not None:
-            value = bool(int(value))
+            if hasattr(value, 'isdigit') and value.isdigit():
+                value = int(value)
+            value = bool(value)
         return value
 
 
@@ -247,3 +251,29 @@ def init_db(app):
     global index_service
     index_service.proxied = IndexService(config=app.config, session=db.session)
     return rv
+
+
+def populate_static(app):
+    with app.app_context():
+        models = import_module("webapp.models")
+        model_names = set([f.split('.csv')[0] for f in os.listdir(STATIC_DATA_DIR) if f.endswith(".csv")])
+        process_order_file = os.path.join(STATIC_DATA_DIR, "_process_order")
+        ordered_model_names = []
+        # process first ordered if exists
+        if os.path.exists(process_order_file):
+            with open(process_order_file) as f:
+                for line in f.readlines():
+                    line = line.strip()
+                    if line:
+                        ordered_model_names.append(line)
+                        model_names.remove(line)
+        for model_name in model_names:
+            ordered_model_names.append(model_name)
+        for model_name in ordered_model_names:
+            model = getattr(models, model_name)
+            with open(os.path.join(STATIC_DATA_DIR, "{}.csv".format(model_name))) as f:
+                reader = csv.DictReader(f)
+                for row in reader:
+                    instance = model(**row)
+                    db.session.add(instance)
+            db.session.commit()