attempt at following a file

oshipka.sh

@@ -8,11 +8,21 @@ echo "oshipka is at: $OSHIPKA_PATH"
 
 #!/usr/bin/env bash
 HELP="
-Usage $0 [ bootstrap ]
+Usage $0 [ bootstrap | worker | web ]
 
     bootstrap [PROJECT_PATH]     Create a new project in PROJECT_PATH
+    worker                       Start worker
+    web                          Start webapp
 "
 
+worker () {
+  celery worker --app=tasks.worker.worker_app --concurrency=1 --loglevel=INFO
+}
+
+web () {
+  python run.py
+}
+
 bootstrap() {
   shift
     PROJECT_PATH=$1
@@ -48,6 +58,10 @@ command_main() {
     INITIAL_COMMAND=$1
     case "$INITIAL_COMMAND" in
         bootstrap) bootstrap "$@"
+        ;;
+        worker) worker "$@"
+        ;;
+      web) web "$@"
         ;;
         *) >&2 echo -e "${HELP}"
         return 1

oshipka/persistance/__init__.py

@@ -11,6 +11,9 @@ def init_db(app):
     app.config["SQLALCHEMY_DATABASE_URI"] = SQLALCHEMY_DATABASE_URI
     app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
 
+    from oshipka.webapp import test_bp
+    app.register_blueprint(test_bp)
+
     db.init_app(app)
     for dir in MAKEDIRS:
         os.makedirs(dir, exist_ok=True)

oshipka/webapp/__init__.py

@@ -1,4 +1,19 @@
-from flask import Flask
+from flask import Flask, Blueprint
+from flask_socketio import SocketIO
 
 app = Flask(__name__)
 app.config["SECRET_KEY"] = "UNSECRET_KEY....478932fjkdsl"
+
+socketio = SocketIO(
+    app,
+    cors_allowed_origins=["http://localhost:5000"],
+)
+
+test_bp = Blueprint('test_bp', __name__,
+                    url_prefix="/test",
+                    template_folder='templates',
+                    static_folder='static',
+                    )
+
+import oshipka.webapp.tasks_routes
+import oshipka.webapp.websockets_routes

oshipka/webapp/tasks_routes.py

@@ -0,0 +1,156 @@
+import json
+import os
+import subprocess
+import threading
+
+from flask import redirect, request, url_for, jsonify, render_template
+from flask_socketio import emit
+from flask_table import Table, LinkCol, Col
+
+from oshipka.persistance import db
+from oshipka.webapp import test_bp, socketio
+from oshipka.worker import worker_app
+
+from config import TASKS_BUF_DIR
+
+
+class Task(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+
+    name = db.Column(db.Unicode)
+    uuid = db.Column(db.Unicode)
+    kwargs = db.Column(db.Unicode)
+
+    def serialize(self):
+        return dict(
+            name=self.name, uuid=self.uuid, kwargs=json.loads(self.kwargs),
+        )
+
+
+class TasksTable(Table):
+    uuid = LinkCol('Task', "test_bp.get_task_status", url_kwargs=dict(task_uuid='uuid'))
+    name = Col('name')
+
+
+def worker_start_task(task_name, task_kwargs):
+    async_task = worker_app.send_task(task_name, [], task_kwargs)
+    task = Task(name=task_name,
+                uuid=async_task.id,
+                kwargs=json.dumps(task_kwargs),
+                )
+    db.session.add(task)
+    db.session.commit()
+    return async_task.id
+
+
+def get_task_ctx(task_uuid):
+    ctx = {}
+    async_task = worker_app.AsyncResult(id=task_uuid)
+    ctx['async_task'] = {
+        'result': async_task.result,
+        'status': async_task.status,
+    }
+    task = Task.query.filter_by(uuid=async_task.id).first()
+    ctx['task'] = task.serialize()
+    out_filename = os.path.join(TASKS_BUF_DIR, task_uuid)
+    if os.path.exists(out_filename):
+        with open(out_filename) as f:
+            ctx['async_task']['stdout'] = f.read()
+    return ctx
+
+
+@test_bp.route("/tasks", methods=["GET"])
+def list_tasks():
+    tasks = Task.query.all()
+    tasks_table = TasksTable(tasks)
+    return render_template("test/tasks.html", tasks_table=tasks_table)
+
+
+@test_bp.route('/tasks/<task_name>/start', methods=['GET', 'POST'])
+def start_task(task_name):
+    task_kwargs = {k: v for k, v in request.form.items() if k != 'csrf_token'}
+    async_task_id = worker_start_task(task_name, task_kwargs)
+    return redirect(url_for('test_bp.get_task_status', task_uuid=async_task_id))
+
+
+@test_bp.route('/tasks/<task_uuid>/status')
+def get_task_status(task_uuid):
+    ctx = get_task_ctx(task_uuid)
+    return render_template('test/task.html', **ctx)
+
+
+@test_bp.route('/tasks/<task_uuid>')
+def get_async_task_result(task_uuid):
+    ctx = get_task_ctx(task_uuid)
+    return jsonify(ctx)
+
+
+PROCESSES = {}
+
+
+def start_tail_process(filename):
+    p = subprocess.Popen(['tail', '-F', filename],
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.STDOUT,
+                         universal_newlines=True,
+                         )
+    return p
+
+
+def tail(process):
+    for line in iter(process.stdout.readline, b''):
+        line = line.strip()
+        print("sending s_tail: {}".format(line))
+        emit("s_tail", {"stdout": line})
+        print("sent s_tail: {}".format((line)))
+
+
+def kill_process(process):
+    if process:
+        # print('process: killing {}'.format(process))
+        process.kill()
+        print('process: killed {}'.format(process))
+
+
+def remove_process(uuid):
+    if uuid in PROCESSES:
+        del PROCESSES[uuid]
+
+
+@socketio.on('connect')
+def handle_connect():
+    sid = str(request.sid)
+    print("socket.connect: session id: {}".format(sid))
+    emit('s_connect', {
+        "sid": sid,
+    })
+
+
+@socketio.on('start_tail')
+def handle_start_tail(json):
+    sid = str(request.sid)
+    task_uuid = json.get("task_uuid")
+    task_out_filename = os.path.join(TASKS_BUF_DIR, task_uuid)
+    process = start_tail_process(task_out_filename)
+    PROCESSES[sid] = ({
+        "sid": sid,
+        "process": process,
+    })
+    print("socket.start_tail: session id: {}".format(sid))
+    emit('s_start_tail', {
+        "sid": sid,
+        "task_uuid": task_uuid,
+    })
+    tail(process)
+
+
+@socketio.on('disconnect')
+def handle_disconnect():
+    sid = str(request.sid)
+    process = PROCESSES.get(sid, {}).get("process")
+    print("socket.disconnect: session id: {}".format(sid))
+    kill_process(process)
+    remove_process(process)
+    emit('s_dissconnect', {
+        "sid": sid,
+    })

oshipka/webapp/templates/test/layout.html

@@ -0,0 +1,178 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <title>Canned</title>
+  <style type="text/css">
+    body {
+      font-family: Open Sans, Arial, sans-serif;
+      color: #444;
+      padding: 0 1em;
+    }
+
+    main {
+      max-width: 1200px;
+      margin: 0 auto;
+    }
+
+    nav {
+      position: fixed;
+      top: 0;
+      background: white;
+      width: 100%;
+    }
+
+    @media (min-width: 1400px) {
+      main {
+        margin: 0 150px;
+      }
+
+      aside {
+        position: fixed;
+        top: 0;
+        right: 0;
+        margin: 0;
+        padding: 1em;
+      }
+
+      #archive-container {
+        display: block !important;
+      }
+
+    }
+
+    input[type=text], textarea {
+      width: 100%;
+      padding: 0.5em;
+      font-family: Open Sans, Arial sans-serif;
+      font-size: 1em;
+      box-sizing: border-box;
+    }
+
+    textarea {
+        height: 5em;
+    }
+
+    .input-label {
+        font-weight: bold;
+    }
+
+    blockquote > h1 {
+      font-size: 2em;
+    }
+
+    blockquote > h2 {
+      font-size: 1.5em;
+    }
+
+    blockquote > h3 {
+      font-size: 1.25em;
+    }
+
+    cite:before {
+      content: "\2014 ";
+    }
+
+    cite {
+      font-size: 0.8em;
+    }
+
+    .search-form {
+      display: inline;
+    }
+
+    .flash {
+      padding: 4px;
+      margin: 4px;
+      top: 1em;
+      position: relative;
+    }
+
+    .flash.error {
+      border: 1px solid #f22;
+      background-color: #faa;
+      color: #f22;
+    }
+
+    .flash.message, .flash.info {
+      border: 1px solid #28f;
+      background-color: #cef;
+      color: #28f;
+    }
+
+    .flash.success {
+      border: 1px solid #2f8;
+      background-color: #cfe;
+      color: #094;
+    }
+
+    .danger {
+      color: #f22;
+    }
+
+    .pull-right {
+      float: right;
+    }
+
+    pre {
+      white-space: pre-wrap;
+    }
+
+    table {
+      border-spacing: 0;
+      border-collapse: collapse;
+    }
+
+    table tr {
+      background-color: #fff;
+      border-top: 1px solid #aaa;
+    }
+
+    table tr:nth-child(2n) {
+      background-color: #f5f5f5;
+    }
+
+    table th, table td {
+      padding: 6px 12px;
+      border: 1px solid #ddd;
+    }
+
+    .vertical-padding {
+      height: 80px;
+    }
+  </style>
+  {% block style %}{% endblock %}
+</head>
+<body>
+<nav>
+  {% include "test/navigation.html" %}
+</nav>
+{% with messages = get_flashed_messages(with_categories=true) %}
+  {% if messages %}
+    {% for category, message in messages %}
+      <div class="flash {{ category }}">{{ message }}</div>
+    {% endfor %}
+  {% endif %}
+{% endwith %}
+<div class="vertical-padding"></div>
+<aside>
+  {% block aside %}{% endblock %}
+</aside>
+<main>
+  {% if ctx and ctx.messages %}
+    {% for message in ctx.messages %}
+      <div class="flash {{ message.type.value }}">
+        <a href="/messages/{{ message.id }}/dismiss"
+           style="float:right;">x</a>
+        {{ message.body|safe }}
+      </div>
+    {% endfor %}
+  {% endif %}
+  {% block content %}{% endblock %}
+</main>
+<script src="{{ url_for('test_bp.static', filename='js/jquery.js') }}"></script>
+{% block script %}{% endblock %}
+</body>
+</html>

oshipka/webapp/templates/test/navigation.html

@@ -0,0 +1,3 @@
+Test
+<a href="{{ url_for('test_bp.websockets') }}">websockets</a> |
+<a href="{{ url_for('test_bp.list_tasks') }}">tasks</a> |

oshipka/webapp/templates/test/task.html

@@ -0,0 +1,45 @@
+{% extends "test/layout.html" %}
+
+{% block aside %}
+
+{% endblock %}
+
+{% block content %}
+    <p><strong>Status:</strong>{{ async_task.status }}</p>
+    <p><strong>Result:</strong>{{ async_task.result }}</p>
+    <p><strong>Task ID:</strong><span id="task_uuid">{{ task.uuid }}</span></p>
+    <p><strong>Task name:</strong>{{ task.name }}</p>
+    <p><strong>Task kwargs:</strong>{{ task.kwargs }}</p>
+    <p><strong>Task out:</strong><span id="task_out"></span></p>
+{% endblock %}
+
+{% block script %}
+    <script src="{{ url_for('test_bp.static', filename='js/socket.io.js') }}"></script>
+    <script>
+        var task_uuid = $('#task_uuid').text();
+
+        function connectServer() {
+            socket = io.connect();
+            socket.on('s_connect', function (e) {
+                console.log("s_connect");
+            });
+
+            socket.emit("start_tail", {task_uuid: task_uuid});
+
+            socket.on('s_start_tail', function (e) {
+                console.log('server start tail', e);
+            });
+
+            socket.on('s_tail', function (o) {
+                console.log('server tail', o);
+                $('#task_out').text(o.stdout);
+            });
+
+            socket.on('s_disconnect', function (e) {
+                console.log('server disconnected', e);
+            });
+        }
+
+        connectServer()
+    </script>
+{% endblock %}

oshipka/webapp/templates/test/tasks.html

@@ -0,0 +1,11 @@
+{% extends "test/layout.html" %}
+
+{% block aside %}
+
+{% endblock %}
+
+{% block content %}
+<a href="{{ url_for('test_bp.start_task', task_name='long_running_task') }}">Start long_running_task</a>
+<hr>
+{{ tasks_table }}
+{% endblock %}

oshipka/webapp/templates/test/websockets.html

@@ -0,0 +1,19 @@
+{% extends "test/layout.html" %}
+
+{% block content %}
+{% endblock %}
+
+{% block script %}
+<script src="{{ url_for('test_bp.static', filename='js/socket.io.js') }}"></script>
+<script>
+    var socket = io();
+    socket.on('connect', function () {
+        console.log("SOCKETIO: sending SYN.");
+        socket.emit('SYN', {data: 'SYN'});
+        console.log("SOCKETIO: sent SYN.");
+    });
+    socket.on('SYN-ACK', function (data) {
+        console.log("SOCKETIO: rcvd SYN-ACK: " + data);
+    });
+</script>
+{% endblock %}

oshipka/webapp/websockets_routes.py

@@ -0,0 +1,17 @@
+from flask import render_template
+from flask_socketio import emit
+
+from oshipka.webapp import test_bp, socketio
+
+
+@test_bp.route('/websockets')
+def websockets():
+    return render_template("test/websockets.html")
+
+
+@socketio.on('SYN')
+def handle_my_custom_namespace_event(json):
+    print('SOCKETIO: rcvd SYN: {}'.format(json))
+    print('SOCKETIO: sending SYN-ACK')
+    emit("SYN-ACK", {"data": 'SYN-ACK'})
+    print('SOCKETIO: sent SYN-ACK')

oshipka/worker.py

@@ -0,0 +1,50 @@
+import os
+import sys
+
+from celery import Celery
+from celery.signals import task_prerun
+
+from config import TASKS_IN_DIR, TASKS_PROC_DIR, TASKS_BUF_DIR
+
+from oshipka.persistance import db
+
+worker_app = Celery(__name__)
+worker_app.conf.update({
+    'broker_url': 'filesystem://',
+    'result_backend': "file://{}".format(TASKS_PROC_DIR),
+    'broker_transport_options': {
+        'data_folder_in': TASKS_IN_DIR,
+        'data_folder_out': TASKS_IN_DIR,
+        'data_folder_processed': TASKS_PROC_DIR,
+    },
+    'imports': ('tasks',),
+    'result_persistent': True,
+    'task_serializer': 'json',
+    'result_serializer': 'json',
+    'accept_content': ['json']})
+
+
+class Unbuffered(object):
+    def __init__(self, stream):
+        self.stream = stream
+
+    def write(self, data):
+        self.stream.write(data)
+        self.stream.flush()
+
+    def writelines(self, datas):
+        self.stream.writelines(datas)
+        self.stream.flush()
+
+    def __getattr__(self, attr):
+        return getattr(self.stream, attr)
+
+
+@task_prerun.connect
+def before_task(task_id=None, task=None, *args, **kwargs):
+    from oshipka import app
+    sys.stdout = Unbuffered(open(os.path.join(TASKS_BUF_DIR, task_id), 'w'))
+    sys.stderr = Unbuffered(open(os.path.join(TASKS_BUF_DIR, task_id), 'w'))
+    db.init_app(app)
+    app.app_context().push()
+    app.test_request_context().push()

requirements.txt

@@ -1,8 +1,28 @@
+amqp==2.5.2
+Babel==2.8.0
+billiard==3.6.3.0
+celery==4.4.2
 click==7.1.1
+dnspython==1.16.0
+eventlet==0.25.1
 Flask==1.1.1
-Flask-SQLAlchemy==2.4.1
+Flask-Babel==1.0.0
+Flask-Celery==2.4.3
+Flask-Script==2.0.6
+Flask-SocketIO==4.2.1
+Flask-Table==0.5.0
+greenlet==0.4.15
+importlib-metadata==1.5.0
 itsdangerous==1.1.0
 Jinja2==2.11.1
+kombu==4.6.8
 MarkupSafe==1.1.1
-SQLAlchemy==1.3.15
+monotonic==1.5
+pkg-resources==0.0.0
+python-engineio==3.12.1
+python-socketio==4.5.0
+pytz==2019.3
+six==1.14.0
+vine==1.3.0
 Werkzeug==1.0.0
+zipp==3.1.0

setup.py

@@ -2,18 +2,8 @@ from setuptools import setup, find_packages
 
 
 def gen_from_file():
-    return [
+    with open("requirements.txt") as f:
-        'click==7.1.1',
+        return [line.strip() for line in f.readlines()]
-        'Flask==1.1.1',
-        'Flask-SQLAlchemy==2.4.1',
-        'itsdangerous==1.1.0',
-        'Jinja2==2.11.1',
-        'MarkupSafe==1.1.1',
-        'SQLAlchemy==1.3.15',
-        'Werkzeug==1.0.0',
-    ]
-    # with open("requirements.txt") as f:
-    #     return [line.strip() for line in f.readlines()]
 
 
 setup(name='oshipka',