search and json file cache

oshipka/persistance/__init__.py

@@ -19,6 +19,7 @@ from sqlalchemy.orm.collections import InstrumentedList
 from sqlalchemy_utils import Choice
 from tww.lib import solve_query, resolve_timezone, dt_tz_translation, time_ago
 from whooshalchemy import IndexService
+from oshipka.search import add_to_index, remove_from_index, query_index
 
 db = SQLAlchemy()
 
@@ -279,3 +280,46 @@ def populate_static(app):
                     instance = model(**row)
                     db.session.add(instance)
             db.session.commit()
+
+
+class SearchableMixin(object):
+    @classmethod
+    def search(cls, expression, page, per_page):
+        ids, total = query_index(cls.__tablename__, expression, page, per_page)
+        if total == 0:
+            return cls.query.filter_by(id=0), 0
+        when = []
+        for i in range(len(ids)):
+            when.append((ids[i], i))
+        return cls.query.filter(cls.id.in_(ids)).order_by(
+            db.case(when, value=cls.id)), total
+
+    @classmethod
+    def before_commit(cls, session):
+        session._changes = {
+            'add': list(session.new),
+            'update': list(session.dirty),
+            'delete': list(session.deleted)
+        }
+
+    @classmethod
+    def after_commit(cls, session):
+        for obj in session._changes['add']:
+            if isinstance(obj, SearchableMixin):
+                add_to_index(obj.__tablename__, obj)
+        for obj in session._changes['update']:
+            if isinstance(obj, SearchableMixin):
+                add_to_index(obj.__tablename__, obj)
+        for obj in session._changes['delete']:
+            if isinstance(obj, SearchableMixin):
+                remove_from_index(obj.__tablename__, obj)
+        session._changes = None
+
+    @classmethod
+    def reindex(cls):
+        for obj in cls.query:
+            add_to_index(cls.__tablename__, obj)
+
+
+db.event.listen(db.session, 'before_commit', SearchableMixin.before_commit)
+db.event.listen(db.session, 'after_commit', SearchableMixin.after_commit)

oshipka/search.py

@@ -0,0 +1,27 @@
+from flask import current_app
+
+
+def add_to_index(index, model):
+    if not hasattr(current_app, 'elasticsearch'):
+        return
+    payload = {}
+    for field in model.__searchable__:
+        payload[field] = getattr(model, field)
+    current_app.elasticsearch.index(index=index, id=model.id, body=payload)
+
+
+def remove_from_index(index, model):
+    if not hasattr(current_app, 'elasticsearch'):
+        return
+    current_app.elasticsearch.delete(index=index, id=model.id)
+
+
+def query_index(index, query, page, per_page):
+    if not hasattr(current_app, 'elasticsearch'):
+        return [], 0
+    search = current_app.elasticsearch.search(
+        index=index,
+        body={'query': {'multi_match': {'query': query, 'fields': ['*']}},
+              'from': (page - 1) * per_page, 'size': per_page})
+    ids = [int(hit['_id']) for hit in search['hits']['hits']]
+    return ids, search['hits']['total']['value']

oshipka/util/process.py

@@ -9,7 +9,7 @@ def process_iterable(iterable: set, process_func, processed: set = None, errors:
     f_args = list() if not f_args else f_args
     f_kwargs = dict() if not f_kwargs else f_kwargs
 
-    to_process = iterable - processed - errors
+    to_process = iterable# - processed - errors
     tot_iterables = len(to_process)
     tot_start = time()
 

oshipka/util/simple_file_cache.py

@@ -0,0 +1,70 @@
+import json
+import os
+from datetime import datetime
+
+from config import CACHE_DIR
+
+DEFAULT_CACHE_DIR = CACHE_DIR
+DEFAULT_CACHE_TIME = float('inf')
+
+
+class file_cache(object):
+    def __init__(self, cache_dir=None, cache_time=None):
+        self.cache_dir = cache_dir or DEFAULT_CACHE_DIR
+        self.cache_time = cache_time or DEFAULT_CACHE_TIME
+
+    def get_cache_file(self, f):
+        my_name = __name__
+        f_name = f.__name__
+        return os.path.join(self.cache_dir, "{}.{}".format(my_name, f_name))
+
+    def get_cache(self, f, args, kwargs):
+        cache_file = self.get_cache_file(f)
+        if os.path.exists(cache_file):
+            with open(cache_file, 'r') as cf:
+                cache_entries = json.load(cf)
+            for cache_entry in cache_entries:
+                if cache_entry.get('args') == list(args) and cache_entry.get('kwargs') == kwargs:
+                    return cache_entry.get('rv')
+        return None
+
+    def set_cache(self, f, args, kwargs, rv):
+        cache_file = self.get_cache_file(f)
+        cache_entries = []
+        if os.path.exists(cache_file):
+            with open(cache_file, 'r') as cf:
+                cache_entries = json.load(cf)
+        for idx, cache_entry in enumerate(cache_entries):
+            if cache_entry.get('args') == list(args) and cache_entry.get('kwargs') == kwargs:
+                cache_entry['rv'] = rv
+                cache_entries[idx] = cache_entry
+                break
+        else:
+            cache_entry = {
+                'args': list(args),
+                'kwargs': kwargs,
+                'rv': rv,
+            }
+            cache_entries.append(cache_entry)
+        with open(cache_file, 'w') as cf:
+            json.dump(cache_entries, cf)
+
+    def __call__(self, f):
+        def wrapped_f(*args, **kwargs):
+            cached_rv = self.get_cache(f, args, kwargs)
+            if cached_rv:
+                return cached_rv
+            rv = f(*args, **kwargs)
+            self.set_cache(f, args, kwargs, rv)
+            return rv
+
+        return wrapped_f
+
+
+@file_cache()
+def main(query):
+    return query + str(datetime.utcnow())
+
+
+if __name__ == "__main__":
+    print(main("helo"))

oshipka/webapp/__init__.py

@@ -2,7 +2,7 @@ import os
 
 from flask import Flask, Blueprint
 
-OSHIPKA_PATH = os.getenv('OSHIPKA_PATH')
+OSHIPKA_PATH = os.getenv('OSHIPKA_PATH', '/home/pi2/workspace/oshipka')
 
 app = Flask(__name__)
 app.config["SECRET_KEY"] = "UNSECRET_KEY....478932fjkdsl"