automating provisioning

.gitignore

@@ -4,3 +4,5 @@ venv
 data/db.sqlite
 __pycache__
 oshipka.egg-info
+provision/tmp
+secrets.py

bootstrap/config.py

@@ -5,7 +5,7 @@ basepath = os.path.dirname(os.path.realpath(__file__))
 DATA_DIR = os.path.join(basepath, "data")
 STATIC_DATA_DIR = os.path.join(basepath, "data_static")
 
-MEDIA_DIR = os.path.join(basepath, "media")
+MEDIA_DIR = os.path.join(DATA_DIR, "media")
 
 TASKS_DIR = os.path.join(DATA_DIR, "tasks")
 TASKS_IN_DIR = os.path.join(TASKS_DIR, "in")

oshipka.sh

@@ -28,6 +28,7 @@ Usage $0 [ bootstrap | model | db_migrate | db_upgrade | db_populate | db_recrea
     link                         Link dev oshipka
 
     prod                         Run in prod
+    prod_install                 Install in prod
     cert [DOMAIN]                Install certificate
 "
 
@@ -69,7 +70,7 @@ install_cert() {
   shift
   PROJECT_DOMAIN=$1
   sudo apt install certbot
-  certbot certonly --authenticator standalone --installer nginx --pre-hook "service nginx stop" --post-hook "service nginx start" --redirect --agree-tos --no-eff-email --email admin@app.com -d ${PROJECT_DOMAIN} --no-bootstrap
+  certbot certonly --authenticator standalone --installer nginx --pre-hook "service nginx stop" --post-hook "service nginx start" --redirect --agree-tos --no-eff-email --email danieltcv@gmail.com -d ${PROJECT_DOMAIN} --no-bootstrap
 }
 
 bootstrap() {
@@ -121,6 +122,57 @@ run_in_prod() {
    gunicorn -w 4 -b 0.0.0.0:${PORT} run:app
 }
 
+prod_install() {
+   shift
+   source venv/bin/activate
+   PROJECT_NAME=$(basename `pwd`)
+   echo "1/4 Generating service and config files..."
+   "${OSHIPKA_PATH}/venv/bin/python" "${OSHIPKA_PATH}/provision/prod_mgmt.py"
+   if [ -f "/etc/systemd/system/${PROJECT_NAME}.service" ]; then
+     echo "Service gunicorn for ${PROJECT_NAME} service exists."
+     systemctl status ${PROJECT_NAME}
+   else
+     echo "Installing '$PROJECT_NAME' gunicorn service"
+     sudo cp "${OSHIPKA_PATH}/provision/tmp/${PROJECT_NAME}.service" /etc/systemd/system/
+     sudo systemctl enable "${PROJECT_NAME}"
+     sudo systemctl start "${PROJECT_NAME}"
+   fi
+
+   echo "2/5 Installing '$PROJECT_NAME' worker service"
+   if [ -f "/etc/systemd/system/${PROJECT_NAME}_worker.service" ]; then
+     echo "Service worker for ${PROJECT_NAME} service exists."
+     systemctl status "${PROJECT_NAME}_worker"
+   else
+     sudo cp "${OSHIPKA_PATH}/provision/tmp/${PROJECT_NAME}_worker.service" /etc/systemd/system/
+     sudo systemctl enable "${PROJECT_NAME}_worker"
+     sudo systemctl start "${PROJECT_NAME}_worker"
+   fi
+
+   # TODO: Update DNS
+   NGINX_CONFIG_FILE=$(basename `find $OSHIPKA_PATH/provision/tmp -name *.conf`)
+   DOMAIN=$(basename -s .conf $NGINX_CONFIG_FILE)
+   echo "3/5 Installing '$DOMAIN' domain..."
+   "${OSHIPKA_PATH}/venv/bin/python" "${OSHIPKA_PATH}/audo_dns/set_domain.py" "$DOMAIN"
+
+   # TODO: Create certificate
+   echo "4/5 Installing '$PROJECT_NAME' certificate..."
+   install_cert $DOMAIN
+
+   echo "5/5 Installing '$PROJECT_NAME' nginx config..."
+   if [ -f "/etc/nginx/sites-available/${NGINX_CONFIG_FILE}" ]; then
+     echo "Nginx config for ${PROJECT_NAME} available."
+     if [ -f "/etc/nginx/sites-enabled/${NGINX_CONFIG_FILE}" ]; then
+       echo "Nginx config for ${PROJECT_NAME} enabled."
+     else
+       echo "Nginx config for ${PROJECT_NAME} NOT enabled."
+     fi
+     else
+       echo "Installing nginx config for ${PROJECT_NAME} -> enabling + available."
+       sudo cp "${OSHIPKA_PATH}/provision/tmp/${NGINX_CONFIG_FILE}" /etc/nginx/sites-enabled/
+       sudo ln -s "/etc/nginx/sites-available/${NGINX_CONFIG_FILE}" "/etc/nginx/sites-enabled/${NGINX_CONFIG_FILE}"
+   fi
+}
+
 model() {
    shift
    source venv/bin/activate
@@ -202,6 +254,8 @@ command_main() {
         ;;
         prod) run_in_prod "$@"
         ;;
+        prod_install) prod_install "$@"
+        ;;
         cert) install_cert "$@"
         ;;
         *) >&2 echo -e "${HELP}"

oshipka/util/os.py

@@ -1,13 +1,21 @@
+import os
+import shlex
 import subprocess
 
 
+def run_os_cmd(command):
+    output = os.popen(command).read()
+    return output
+
+
 def run_cmd(command):
-    process = subprocess.Popen(command.split(),
+    process = subprocess.Popen(shlex.split(command),
                                stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT,
                                universal_newlines=True,
                                )
-    return '\n'.join([str(line) for line in iter(process.stdout.readline, '')])
+    for line in iter(process.stdout.readline, ''):
+        yield line
 
 
 def print_cmd(command):

oshipka/util/process.py

@@ -1,5 +1,36 @@
 import traceback
-from time import time
+from time import time, sleep
+
+
+def process_exp_backoff(func, func_args=None, func_kwargs=None, func_exc_classes=None,
+                        max_attempts=3, initial_sleep_time=1, sleep_exp=2, max_sleep_time=0, timeout=0):
+    func_args = list() if func_args is None else func_args
+    func_kwargs = dict() if func_kwargs is None else func_kwargs
+    func_exc_classes = Exception if func_exc_classes is None else func_exc_classes
+    attempt, start_time, sleep_time = 1, time(), initial_sleep_time
+    while True:
+        try:
+            print("EXP_BACKOFF: Attempt {}/{}".format(attempt, max_attempts))
+            return func(*func_args, **func_kwargs)
+        except func_exc_classes as e:
+            print("EXP_BACKOFF: Received error at attempt: {}/{}".format(attempt, max_attempts))
+            traceback.print_exc()
+            print(e)
+        total_time = time() - start_time
+        if (max_sleep_time and sleep_time >= max_sleep_time) or \
+                (max_attempts and attempt >= max_attempts) or \
+                (timeout and timeout >= total_time):
+            raise Exception("EXP_BACKOFF: Giving up: \n"
+                            "- sleep_time: {}/{}\n"
+                            "- attempt: {}/{}\n"
+                            "- total_time: {}/{}\n".format(
+                sleep_time, max_sleep_time,
+                attempt, max_attempts,
+                total_time, timeout))
+        sleep_time *= sleep_exp
+        print("EXP_BACKOFF: Sleeping {}...".format(sleep_time))
+        sleep(sleep_time)
+        attempt += 1
 
 
 def process_iterable(iterable: set, process_func, processed: set = None, errors: set = None,
@@ -9,7 +40,7 @@ def process_iterable(iterable: set, process_func, processed: set = None, errors:
     f_args = list() if not f_args else f_args
     f_kwargs = dict() if not f_kwargs else f_kwargs
 
-    to_process = iterable# - processed - errors
+    to_process = iterable  # - processed - errors
     tot_iterables = len(to_process)
     tot_start = time()
 

provision/auto_dns/auto_dns.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=auto dns
+Requires=network.target
+After=network.target
+
+[Service]
+Type=simple
+User={{ user }}
+Type=simple
+WorkingDirectory={{ oshipka_path }}
+ExecStart=/bin/bash -c '{{ oshipka_path }}/venv/bin/python {{ oshipka_path }}/provision/auto_dns/check.py'
+Restart=on-failure
+
+[Install]
+WantedBy=default.target

provision/auto_dns/check.py

@@ -0,0 +1,172 @@
+import csv
+import ipaddress
+import threading
+from collections import defaultdict
+from datetime import datetime
+from time import sleep
+import sys
+
+import requests
+from lexicon.client import Client
+from lexicon.config import ConfigResolver
+
+from oshipka.util.process import process_exp_backoff
+from secrets import HOVER_USERNAME, HOVER_PASSWORD
+
+SLEEP_SECONDS = 60 * 30  # 30 min
+
+ipv4_sites = [
+    "http://www.icanhazip.com",
+    "http://ipecho.net/plain",
+    "http://checkip.amazonaws.com",
+    "http://ipinfo.io/ip",
+    "http://ifconfig.me"
+]
+
+ipv6_sites = [
+    "http://bot.whatismyipaddress.com",
+    "https://diagnostic.opendns.com/myip",
+    "https://ifconfig.co/ip",
+]
+
+dns_to_check = []
+with open('dns_addresses.csv') as csvf:
+    csvreader = csv.reader(csvf)
+    for row in csvreader:
+        dns_to_check.append(row)
+
+ip_checkers = ipv4_sites + ipv6_sites
+
+results = []
+real = defaultdict(dict)
+
+
+def test_ip(ip):
+    try:
+        ipaddress_info = ipaddress.ip_address(ip)
+    except ValueError as e:
+        print(e)
+        return None
+    if ipaddress_info.version == 4:
+        version = 'A'
+    elif ipaddress_info.version == 6:
+        version = 'AAAA'
+    else:
+        return None
+    return {
+        "addr": ipaddress_info.exploded,
+        "version": version,
+    }
+
+
+def fetch_ip(url):
+    try:
+        resp = requests.get(url)
+        ip = resp.text.strip()
+        ip_info = test_ip(ip)
+        if ip_info:
+            results.append(ip_info)
+    except Exception as e:
+        print("Error fetching {}: {}".format(url, e))
+
+
+tso, tsd = [], []
+
+
+def check_observed():
+    for url in ip_checkers:
+        t = threading.Thread(target=fetch_ip, args=(url,))
+        tso.append(t)
+        t.start()
+
+
+def check_dns():
+    for domain, ipv in dns_to_check:
+        t = threading.Thread(target=get_dns, args=(domain, ipv,))
+        tsd.append(t)
+        t.start()
+
+
+def get_hover_results(domain, ipv):
+    dns_config = ConfigResolver()
+    dns_config.with_dict({
+        'provider_name': 'hover',
+        'action': 'list',
+        'type': ipv,
+        'domain': domain,
+        'hover': {
+            'auth_username': HOVER_USERNAME,
+            'auth_password': HOVER_PASSWORD,
+        }
+    })
+    client = Client(dns_config)
+    return client.execute()
+
+
+def set_hover_results(domain, ipv, new):
+    dns_config = ConfigResolver()
+    dns_config.with_dict({
+        'provider_name': 'hover',
+        'action': 'update',
+        'type': ipv,
+        'domain': domain,
+        'name': domain,
+        'content': new,
+        'hover': {
+            'auth_username': HOVER_USERNAME,
+            'auth_password': HOVER_PASSWORD,
+        }
+    })
+    client = Client(dns_config)
+    return client.execute()
+
+
+def get_dns(domain, ipv):
+    results = process_exp_backoff(get_hover_results, func_args=[domain, ipv])
+    res = [x.get('content') for x in results if x.get('name') == domain]
+    if len(res) == 1:
+        print("Real {}: {}".format(ipv, res[0]))
+        real[domain][ipv] = res[0]
+
+
+def set_dns(domain, ipv, new):
+    return process_exp_backoff(set_hover_results, func_args=[domain, ipv, new])
+
+
+def get_uniq(_list):
+    if len(_list) > 1 and len(set(_list)) == 1:
+        return _list[0]
+
+
+def main():
+    check_dns()
+    check_observed()
+    for t in tso:
+        t.join()
+    for t in tsd:
+        t.join()
+    for domain, version in dns_to_check:
+        ip_obs = get_uniq([x['addr'] for x in results if x['version'] == version])
+        print("{}/{}: {}".format(domain, version, ip_obs))
+        ip_real = real.get(domain).get(version)
+        sys.stdout.flush()
+        if ip_obs == ip_real:
+            print('{}/{} is same!'.format(domain, version))
+        else:
+            print("{}/{} diff on dns: real: {}, obs: {}".format(domain, version, ip_real, ip_obs))
+            if set_dns(domain, version, ip_real):
+                print("update successful {}/{} -> {}".format(domain, version, ip_real))
+            else:
+                print("update failed: {}/{} on dns is {}, but obs is {}".format(
+                    domain, version, ip_real, ip_obs
+                ))
+
+
+if __name__ == "__main__":
+    while True:
+        print("{}: Waking up".format(datetime.utcnow()))
+        sys.stdout.flush()
+        main()
+        print("{}: Sleeping for {}".format(datetime.utcnow(), SLEEP_SECONDS))
+        sys.stdout.flush()
+        sleep(SLEEP_SECONDS)

provision/auto_dns/dns_addresses.csv

@@ -0,0 +1,2 @@
+oc.pi2.dev,A
+oc.pi2.dev,AAAA

provision/auto_dns/get_my_ip.py

@@ -0,0 +1,6 @@
+import sys
+
+
+if __name__ == "__main__":
+    domain = sys.argv[1]
+

provision/gunicorn.service

@@ -1,14 +0,0 @@
-[Unit]
-Description=gunicorn service
-Requires=network.target
-After=network.target
-
-[Service]
-User=pi2
-Type=simple
-WorkingDirectory=/home/pi2/watch
-ExecStart=/bin/bash -c 'export OSHIPKA_PATH=/home/pi2/oshipka; $OSHIPKA_PATH/oshipka.sh prod 5000'
-Restart=on-failure
-
-[Install]
-WantedBy=default.target

provision/prod_mgmt.py

@@ -0,0 +1,56 @@
+import os
+
+from jinja2 import FileSystemLoader, Environment
+
+from oshipka.util.os import run_cmd
+from provision.util import find_port, find_private_ipv4
+
+USER = "pi2"
+PARENT_DOMAIN = "pi2.dev"
+
+
+oshipka_path = os.environ.get('OSHIPKA_PATH')
+TEMPLATES_PATH = os.path.join(oshipka_path, "provision", "templates")
+TMP_PATH = os.path.join(oshipka_path, "provision", "tmp")
+os.makedirs(TMP_PATH, exist_ok=True)
+env = Environment(
+    loader=FileSystemLoader(searchpath=TEMPLATES_PATH),
+)
+
+
+def prod_install():
+    pwd = next(run_cmd('pwd')).strip()
+
+    project_name = os.path.basename(pwd)
+    project_domain = "{}.{}".format(project_name, PARENT_DOMAIN)
+    port = find_port()
+    upstream_ip = find_private_ipv4()
+
+    ctx = dict(
+        user=USER,
+        pwd=pwd,
+        oshipka_path=oshipka_path,
+        project_name=project_name,
+        project_domain=project_domain,
+        upstream_ip=upstream_ip,
+        port=port,
+    )
+
+    service_tmpl = env.get_template('gunicorn.service')
+    service_txt = service_tmpl.render(ctx)
+    with open(os.path.join(TMP_PATH, "{}.service".format(project_name)), 'w') as f:
+        f.write(service_txt)
+
+    worker_service_tmpl = env.get_template('worker.service')
+    worker_service_txt = worker_service_tmpl.render(ctx)
+    with open(os.path.join(TMP_PATH, "{}_worker.service".format(project_name)), 'w') as f:
+        f.write(worker_service_txt)
+
+    nginx_tmpl = env.get_template('nginx.conf')
+    nginx_txt = nginx_tmpl.render(ctx)
+    with open(os.path.join(TMP_PATH, "{}.conf".format(project_domain)), 'w') as f:
+        f.write(nginx_txt)
+
+
+if __name__ == "__main__":
+    prod_install()

provision/templates/gunicorn.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=oshipka {{ project_name }} gunicorn service
+Requires=network.target
+After=network.target
+
+[Service]
+User={{ user }}
+Type=simple
+WorkingDirectory={{ pwd }}
+ExecStart=/bin/bash -c 'export OSHIPKA_PATH={{ oshipka_path }}; $OSHIPKA_PATH/oshipka.sh prod {{ port }}'
+Restart=on-failure
+
+[Install]
+WantedBy=default.target

provision/templates/nginx.conf

@@ -2,7 +2,7 @@ server {
     listen 80;
     listen [::]:80;
 
-    server_name PROJECT_DOMAIN;
+    server_name {{ project_domain }};
 
     # redirect all HTTP requests to HTTPS with a 301 Moved Permanently response.
     return 301 https://;
@@ -12,15 +12,15 @@ server {
     listen 443 ssl http2;
     listen [::]:443 ssl http2;
 
-    server_name PROJECT_DOMAIN;
+    server_name {{ project_domain }};
 
     server_tokens off;
     charset utf-8;
-    client_max_body_size 1g;
+    client_max_body_size {{ project_domain }};
 
     # certs sent to the client in SERVER HELLO are concatenated in ssl_certificate
-    ssl_certificate /etc/letsencrypt/live/PROJECT_DOMAIN/fullchain.pem;
+    ssl_certificate /etc/letsencrypt/live/{{ project_domain }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/PROJECT_DOMAIN/privkey.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ project_domain }}/privkey.pem;
     ssl_session_timeout 1d;
     ssl_session_cache shared:MozSSL:10m;  # about 40000 sessions
     ssl_session_tickets off;
@@ -44,10 +44,10 @@ server {
 
     # replace with the IP address of your resolver
     location / {
-        proxy_pass http://192.168.1.112:5001;
+        proxy_pass http://{{ upstream_ip }}:{{ port }};
         proxy_redirect off;
 
-	proxy_set_header Host $host;
+	    proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 

provision/templates/worker.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=oshipka {{ project_name }} worker service
+Requires=network.target
+After=network.target
+
+[Service]
+User={{ user }}
+Type=simple
+WorkingDirectory={{ pwd }}
+ExecStart=/bin/bash -c 'export OSHIPKA_PATH={{ oshipka_path }}; $OSHIPKA_PATH/oshipka.sh worker'
+Restart=on-failure
+
+[Install]
+WantedBy=default.target

provision/util.py

@@ -0,0 +1,27 @@
+import re
+
+from oshipka.util.os import run_os_cmd
+
+PORT = 5000
+
+ipv4_regex = re.compile('^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$')
+
+
+def find_port():
+    for i in range(16):
+        port = PORT + i
+        has_port = run_os_cmd("ss -tl | grep {}".format(port))
+        if not has_port:
+            return port
+
+
+def find_private_ipv4():
+    private_ips = run_os_cmd("hostname -I").split()
+    for private_ip in private_ips:
+        private_ip = private_ip.strip()
+        if ipv4_regex.match(private_ip):
+            return private_ip
+
+
+if __name__ == "__main__":
+    print(find_port())

requirements.txt

@@ -4,6 +4,7 @@ bcrypt==3.1.7
 blinker==1.4
 cffi==1.14.0
 click==7.1.1
+dns-lexicon==3.3.11
 filelock==3.0.12
 Flask==1.1.1
 Flask-Babel==1.0.0