From c4931463228b613818903e44a854f1f2f2a52955 Mon Sep 17 00:00:00 2001 From: Daniel Tsvetkov Date: Tue, 2 Jun 2020 08:59:31 +0200 Subject: [PATCH] search and json file cache --- oshipka/persistance/__init__.py | 44 +++++++++++++++++++ oshipka/search.py | 27 ++++++++++++ oshipka/util/process.py | 2 +- oshipka/util/simple_file_cache.py | 70 +++++++++++++++++++++++++++++++ oshipka/webapp/__init__.py | 2 +- 5 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 oshipka/search.py create mode 100644 oshipka/util/simple_file_cache.py diff --git a/oshipka/persistance/__init__.py b/oshipka/persistance/__init__.py index 77a099f..7c56a0c 100644 --- a/oshipka/persistance/__init__.py +++ b/oshipka/persistance/__init__.py @@ -19,6 +19,7 @@ from sqlalchemy.orm.collections import InstrumentedList from sqlalchemy_utils import Choice from tww.lib import solve_query, resolve_timezone, dt_tz_translation, time_ago from whooshalchemy import IndexService +from oshipka.search import add_to_index, remove_from_index, query_index db = SQLAlchemy() @@ -279,3 +280,46 @@ def populate_static(app): instance = model(**row) db.session.add(instance) db.session.commit() + + +class SearchableMixin(object): + @classmethod + def search(cls, expression, page, per_page): + ids, total = query_index(cls.__tablename__, expression, page, per_page) + if total == 0: + return cls.query.filter_by(id=0), 0 + when = [] + for i in range(len(ids)): + when.append((ids[i], i)) + return cls.query.filter(cls.id.in_(ids)).order_by( + db.case(when, value=cls.id)), total + + @classmethod + def before_commit(cls, session): + session._changes = { + 'add': list(session.new), + 'update': list(session.dirty), + 'delete': list(session.deleted) + } + + @classmethod + def after_commit(cls, session): + for obj in session._changes['add']: + if isinstance(obj, SearchableMixin): + add_to_index(obj.__tablename__, obj) + for obj in session._changes['update']: + if isinstance(obj, SearchableMixin): + add_to_index(obj.__tablename__, obj) + for obj in session._changes['delete']: + if isinstance(obj, SearchableMixin): + remove_from_index(obj.__tablename__, obj) + session._changes = None + + @classmethod + def reindex(cls): + for obj in cls.query: + add_to_index(cls.__tablename__, obj) + + +db.event.listen(db.session, 'before_commit', SearchableMixin.before_commit) +db.event.listen(db.session, 'after_commit', SearchableMixin.after_commit) diff --git a/oshipka/search.py b/oshipka/search.py new file mode 100644 index 0000000..9e564ce --- /dev/null +++ b/oshipka/search.py @@ -0,0 +1,27 @@ +from flask import current_app + + +def add_to_index(index, model): + if not hasattr(current_app, 'elasticsearch'): + return + payload = {} + for field in model.__searchable__: + payload[field] = getattr(model, field) + current_app.elasticsearch.index(index=index, id=model.id, body=payload) + + +def remove_from_index(index, model): + if not hasattr(current_app, 'elasticsearch'): + return + current_app.elasticsearch.delete(index=index, id=model.id) + + +def query_index(index, query, page, per_page): + if not hasattr(current_app, 'elasticsearch'): + return [], 0 + search = current_app.elasticsearch.search( + index=index, + body={'query': {'multi_match': {'query': query, 'fields': ['*']}}, + 'from': (page - 1) * per_page, 'size': per_page}) + ids = [int(hit['_id']) for hit in search['hits']['hits']] + return ids, search['hits']['total']['value'] diff --git a/oshipka/util/process.py b/oshipka/util/process.py index e44cf20..4cf1fdd 100644 --- a/oshipka/util/process.py +++ b/oshipka/util/process.py @@ -9,7 +9,7 @@ def process_iterable(iterable: set, process_func, processed: set = None, errors: f_args = list() if not f_args else f_args f_kwargs = dict() if not f_kwargs else f_kwargs - to_process = iterable - processed - errors + to_process = iterable# - processed - errors tot_iterables = len(to_process) tot_start = time() diff --git a/oshipka/util/simple_file_cache.py b/oshipka/util/simple_file_cache.py new file mode 100644 index 0000000..d410677 --- /dev/null +++ b/oshipka/util/simple_file_cache.py @@ -0,0 +1,70 @@ +import json +import os +from datetime import datetime + +from config import CACHE_DIR + +DEFAULT_CACHE_DIR = CACHE_DIR +DEFAULT_CACHE_TIME = float('inf') + + +class file_cache(object): + def __init__(self, cache_dir=None, cache_time=None): + self.cache_dir = cache_dir or DEFAULT_CACHE_DIR + self.cache_time = cache_time or DEFAULT_CACHE_TIME + + def get_cache_file(self, f): + my_name = __name__ + f_name = f.__name__ + return os.path.join(self.cache_dir, "{}.{}".format(my_name, f_name)) + + def get_cache(self, f, args, kwargs): + cache_file = self.get_cache_file(f) + if os.path.exists(cache_file): + with open(cache_file, 'r') as cf: + cache_entries = json.load(cf) + for cache_entry in cache_entries: + if cache_entry.get('args') == list(args) and cache_entry.get('kwargs') == kwargs: + return cache_entry.get('rv') + return None + + def set_cache(self, f, args, kwargs, rv): + cache_file = self.get_cache_file(f) + cache_entries = [] + if os.path.exists(cache_file): + with open(cache_file, 'r') as cf: + cache_entries = json.load(cf) + for idx, cache_entry in enumerate(cache_entries): + if cache_entry.get('args') == list(args) and cache_entry.get('kwargs') == kwargs: + cache_entry['rv'] = rv + cache_entries[idx] = cache_entry + break + else: + cache_entry = { + 'args': list(args), + 'kwargs': kwargs, + 'rv': rv, + } + cache_entries.append(cache_entry) + with open(cache_file, 'w') as cf: + json.dump(cache_entries, cf) + + def __call__(self, f): + def wrapped_f(*args, **kwargs): + cached_rv = self.get_cache(f, args, kwargs) + if cached_rv: + return cached_rv + rv = f(*args, **kwargs) + self.set_cache(f, args, kwargs, rv) + return rv + + return wrapped_f + + +@file_cache() +def main(query): + return query + str(datetime.utcnow()) + + +if __name__ == "__main__": + print(main("helo")) diff --git a/oshipka/webapp/__init__.py b/oshipka/webapp/__init__.py index 8ada7f2..f046880 100644 --- a/oshipka/webapp/__init__.py +++ b/oshipka/webapp/__init__.py @@ -2,7 +2,7 @@ import os from flask import Flask, Blueprint -OSHIPKA_PATH = os.getenv('OSHIPKA_PATH') +OSHIPKA_PATH = os.getenv('OSHIPKA_PATH', '/home/pi2/workspace/oshipka') app = Flask(__name__) app.config["SECRET_KEY"] = "UNSECRET_KEY....478932fjkdsl"