From 03c2addf8abeda5fe769ab6d9d01435b284bb108 Mon Sep 17 00:00:00 2001 From: Daniel Tsvetkov Date: Sun, 21 Jun 2020 11:17:34 +0200 Subject: [PATCH] automating provisioning --- .gitignore | 4 +- bootstrap/config.py | 2 +- oshipka.sh | 56 ++++++++- oshipka/util/os.py | 14 ++- oshipka/util/process.py | 35 +++++- provision/__init__.py | 0 provision/auto_dns/__init__.py | 0 provision/auto_dns/auto_dns.service | 15 +++ provision/auto_dns/check.py | 172 +++++++++++++++++++++++++++ provision/auto_dns/dns_addresses.csv | 2 + provision/auto_dns/get_my_ip.py | 6 + provision/gunicorn.service | 14 --- provision/prod_mgmt.py | 56 +++++++++ provision/templates/gunicorn.service | 14 +++ provision/{ => templates}/nginx.conf | 14 +-- provision/templates/worker.service | 14 +++ provision/util.py | 27 +++++ requirements.txt | 1 + 18 files changed, 417 insertions(+), 29 deletions(-) create mode 100644 provision/__init__.py create mode 100644 provision/auto_dns/__init__.py create mode 100644 provision/auto_dns/auto_dns.service create mode 100644 provision/auto_dns/check.py create mode 100644 provision/auto_dns/dns_addresses.csv create mode 100644 provision/auto_dns/get_my_ip.py delete mode 100644 provision/gunicorn.service create mode 100644 provision/prod_mgmt.py create mode 100644 provision/templates/gunicorn.service rename provision/{ => templates}/nginx.conf (79%) create mode 100644 provision/templates/worker.service create mode 100644 provision/util.py diff --git a/.gitignore b/.gitignore index db1dc02..69df644 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ venv *.pyc data/db.sqlite __pycache__ -oshipka.egg-info \ No newline at end of file +oshipka.egg-info +provision/tmp +secrets.py \ No newline at end of file diff --git a/bootstrap/config.py b/bootstrap/config.py index c5d3be8..22140f2 100644 --- a/bootstrap/config.py +++ b/bootstrap/config.py @@ -5,7 +5,7 @@ basepath = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(basepath, "data") STATIC_DATA_DIR = os.path.join(basepath, "data_static") -MEDIA_DIR = os.path.join(basepath, "media") +MEDIA_DIR = os.path.join(DATA_DIR, "media") TASKS_DIR = os.path.join(DATA_DIR, "tasks") TASKS_IN_DIR = os.path.join(TASKS_DIR, "in") diff --git a/oshipka.sh b/oshipka.sh index 0b02509..777ce15 100755 --- a/oshipka.sh +++ b/oshipka.sh @@ -28,6 +28,7 @@ Usage $0 [ bootstrap | model | db_migrate | db_upgrade | db_populate | db_recrea link Link dev oshipka prod Run in prod + prod_install Install in prod cert [DOMAIN] Install certificate " @@ -69,7 +70,7 @@ install_cert() { shift PROJECT_DOMAIN=$1 sudo apt install certbot - certbot certonly --authenticator standalone --installer nginx --pre-hook "service nginx stop" --post-hook "service nginx start" --redirect --agree-tos --no-eff-email --email admin@app.com -d ${PROJECT_DOMAIN} --no-bootstrap + certbot certonly --authenticator standalone --installer nginx --pre-hook "service nginx stop" --post-hook "service nginx start" --redirect --agree-tos --no-eff-email --email danieltcv@gmail.com -d ${PROJECT_DOMAIN} --no-bootstrap } bootstrap() { @@ -121,6 +122,57 @@ run_in_prod() { gunicorn -w 4 -b 0.0.0.0:${PORT} run:app } +prod_install() { + shift + source venv/bin/activate + PROJECT_NAME=$(basename `pwd`) + echo "1/4 Generating service and config files..." + "${OSHIPKA_PATH}/venv/bin/python" "${OSHIPKA_PATH}/provision/prod_mgmt.py" + if [ -f "/etc/systemd/system/${PROJECT_NAME}.service" ]; then + echo "Service gunicorn for ${PROJECT_NAME} service exists." + systemctl status ${PROJECT_NAME} + else + echo "Installing '$PROJECT_NAME' gunicorn service" + sudo cp "${OSHIPKA_PATH}/provision/tmp/${PROJECT_NAME}.service" /etc/systemd/system/ + sudo systemctl enable "${PROJECT_NAME}" + sudo systemctl start "${PROJECT_NAME}" + fi + + echo "2/5 Installing '$PROJECT_NAME' worker service" + if [ -f "/etc/systemd/system/${PROJECT_NAME}_worker.service" ]; then + echo "Service worker for ${PROJECT_NAME} service exists." + systemctl status "${PROJECT_NAME}_worker" + else + sudo cp "${OSHIPKA_PATH}/provision/tmp/${PROJECT_NAME}_worker.service" /etc/systemd/system/ + sudo systemctl enable "${PROJECT_NAME}_worker" + sudo systemctl start "${PROJECT_NAME}_worker" + fi + + # TODO: Update DNS + NGINX_CONFIG_FILE=$(basename `find $OSHIPKA_PATH/provision/tmp -name *.conf`) + DOMAIN=$(basename -s .conf $NGINX_CONFIG_FILE) + echo "3/5 Installing '$DOMAIN' domain..." + "${OSHIPKA_PATH}/venv/bin/python" "${OSHIPKA_PATH}/audo_dns/set_domain.py" "$DOMAIN" + + # TODO: Create certificate + echo "4/5 Installing '$PROJECT_NAME' certificate..." + install_cert $DOMAIN + + echo "5/5 Installing '$PROJECT_NAME' nginx config..." + if [ -f "/etc/nginx/sites-available/${NGINX_CONFIG_FILE}" ]; then + echo "Nginx config for ${PROJECT_NAME} available." + if [ -f "/etc/nginx/sites-enabled/${NGINX_CONFIG_FILE}" ]; then + echo "Nginx config for ${PROJECT_NAME} enabled." + else + echo "Nginx config for ${PROJECT_NAME} NOT enabled." + fi + else + echo "Installing nginx config for ${PROJECT_NAME} -> enabling + available." + sudo cp "${OSHIPKA_PATH}/provision/tmp/${NGINX_CONFIG_FILE}" /etc/nginx/sites-enabled/ + sudo ln -s "/etc/nginx/sites-available/${NGINX_CONFIG_FILE}" "/etc/nginx/sites-enabled/${NGINX_CONFIG_FILE}" + fi +} + model() { shift source venv/bin/activate @@ -202,6 +254,8 @@ command_main() { ;; prod) run_in_prod "$@" ;; + prod_install) prod_install "$@" + ;; cert) install_cert "$@" ;; *) >&2 echo -e "${HELP}" diff --git a/oshipka/util/os.py b/oshipka/util/os.py index 637e8b3..fd2f413 100644 --- a/oshipka/util/os.py +++ b/oshipka/util/os.py @@ -1,13 +1,21 @@ +import os +import shlex import subprocess +def run_os_cmd(command): + output = os.popen(command).read() + return output + + def run_cmd(command): - process = subprocess.Popen(command.split(), + process = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, ) - return '\n'.join([str(line) for line in iter(process.stdout.readline, '')]) + for line in iter(process.stdout.readline, ''): + yield line def print_cmd(command): @@ -16,4 +24,4 @@ def print_cmd(command): if __name__ == "__main__": - print_cmd("env") \ No newline at end of file + print_cmd("env") diff --git a/oshipka/util/process.py b/oshipka/util/process.py index 4cf1fdd..b6de7d3 100644 --- a/oshipka/util/process.py +++ b/oshipka/util/process.py @@ -1,5 +1,36 @@ import traceback -from time import time +from time import time, sleep + + +def process_exp_backoff(func, func_args=None, func_kwargs=None, func_exc_classes=None, + max_attempts=3, initial_sleep_time=1, sleep_exp=2, max_sleep_time=0, timeout=0): + func_args = list() if func_args is None else func_args + func_kwargs = dict() if func_kwargs is None else func_kwargs + func_exc_classes = Exception if func_exc_classes is None else func_exc_classes + attempt, start_time, sleep_time = 1, time(), initial_sleep_time + while True: + try: + print("EXP_BACKOFF: Attempt {}/{}".format(attempt, max_attempts)) + return func(*func_args, **func_kwargs) + except func_exc_classes as e: + print("EXP_BACKOFF: Received error at attempt: {}/{}".format(attempt, max_attempts)) + traceback.print_exc() + print(e) + total_time = time() - start_time + if (max_sleep_time and sleep_time >= max_sleep_time) or \ + (max_attempts and attempt >= max_attempts) or \ + (timeout and timeout >= total_time): + raise Exception("EXP_BACKOFF: Giving up: \n" + "- sleep_time: {}/{}\n" + "- attempt: {}/{}\n" + "- total_time: {}/{}\n".format( + sleep_time, max_sleep_time, + attempt, max_attempts, + total_time, timeout)) + sleep_time *= sleep_exp + print("EXP_BACKOFF: Sleeping {}...".format(sleep_time)) + sleep(sleep_time) + attempt += 1 def process_iterable(iterable: set, process_func, processed: set = None, errors: set = None, @@ -9,7 +40,7 @@ def process_iterable(iterable: set, process_func, processed: set = None, errors: f_args = list() if not f_args else f_args f_kwargs = dict() if not f_kwargs else f_kwargs - to_process = iterable# - processed - errors + to_process = iterable # - processed - errors tot_iterables = len(to_process) tot_start = time() diff --git a/provision/__init__.py b/provision/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/provision/auto_dns/__init__.py b/provision/auto_dns/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/provision/auto_dns/auto_dns.service b/provision/auto_dns/auto_dns.service new file mode 100644 index 0000000..497b15a --- /dev/null +++ b/provision/auto_dns/auto_dns.service @@ -0,0 +1,15 @@ +[Unit] +Description=auto dns +Requires=network.target +After=network.target + +[Service] +Type=simple +User={{ user }} +Type=simple +WorkingDirectory={{ oshipka_path }} +ExecStart=/bin/bash -c '{{ oshipka_path }}/venv/bin/python {{ oshipka_path }}/provision/auto_dns/check.py' +Restart=on-failure + +[Install] +WantedBy=default.target diff --git a/provision/auto_dns/check.py b/provision/auto_dns/check.py new file mode 100644 index 0000000..a96d584 --- /dev/null +++ b/provision/auto_dns/check.py @@ -0,0 +1,172 @@ +import csv +import ipaddress +import threading +from collections import defaultdict +from datetime import datetime +from time import sleep +import sys + +import requests +from lexicon.client import Client +from lexicon.config import ConfigResolver + +from oshipka.util.process import process_exp_backoff +from secrets import HOVER_USERNAME, HOVER_PASSWORD + +SLEEP_SECONDS = 60 * 30 # 30 min + +ipv4_sites = [ + "http://www.icanhazip.com", + "http://ipecho.net/plain", + "http://checkip.amazonaws.com", + "http://ipinfo.io/ip", + "http://ifconfig.me" +] + +ipv6_sites = [ + "http://bot.whatismyipaddress.com", + "https://diagnostic.opendns.com/myip", + "https://ifconfig.co/ip", +] + +dns_to_check = [] +with open('dns_addresses.csv') as csvf: + csvreader = csv.reader(csvf) + for row in csvreader: + dns_to_check.append(row) + +ip_checkers = ipv4_sites + ipv6_sites + +results = [] +real = defaultdict(dict) + + +def test_ip(ip): + try: + ipaddress_info = ipaddress.ip_address(ip) + except ValueError as e: + print(e) + return None + if ipaddress_info.version == 4: + version = 'A' + elif ipaddress_info.version == 6: + version = 'AAAA' + else: + return None + return { + "addr": ipaddress_info.exploded, + "version": version, + } + + +def fetch_ip(url): + try: + resp = requests.get(url) + ip = resp.text.strip() + ip_info = test_ip(ip) + if ip_info: + results.append(ip_info) + except Exception as e: + print("Error fetching {}: {}".format(url, e)) + + +tso, tsd = [], [] + + +def check_observed(): + for url in ip_checkers: + t = threading.Thread(target=fetch_ip, args=(url,)) + tso.append(t) + t.start() + + +def check_dns(): + for domain, ipv in dns_to_check: + t = threading.Thread(target=get_dns, args=(domain, ipv,)) + tsd.append(t) + t.start() + + +def get_hover_results(domain, ipv): + dns_config = ConfigResolver() + dns_config.with_dict({ + 'provider_name': 'hover', + 'action': 'list', + 'type': ipv, + 'domain': domain, + 'hover': { + 'auth_username': HOVER_USERNAME, + 'auth_password': HOVER_PASSWORD, + } + }) + client = Client(dns_config) + return client.execute() + + +def set_hover_results(domain, ipv, new): + dns_config = ConfigResolver() + dns_config.with_dict({ + 'provider_name': 'hover', + 'action': 'update', + 'type': ipv, + 'domain': domain, + 'name': domain, + 'content': new, + 'hover': { + 'auth_username': HOVER_USERNAME, + 'auth_password': HOVER_PASSWORD, + } + }) + client = Client(dns_config) + return client.execute() + + +def get_dns(domain, ipv): + results = process_exp_backoff(get_hover_results, func_args=[domain, ipv]) + res = [x.get('content') for x in results if x.get('name') == domain] + if len(res) == 1: + print("Real {}: {}".format(ipv, res[0])) + real[domain][ipv] = res[0] + + +def set_dns(domain, ipv, new): + return process_exp_backoff(set_hover_results, func_args=[domain, ipv, new]) + + +def get_uniq(_list): + if len(_list) > 1 and len(set(_list)) == 1: + return _list[0] + + +def main(): + check_dns() + check_observed() + for t in tso: + t.join() + for t in tsd: + t.join() + for domain, version in dns_to_check: + ip_obs = get_uniq([x['addr'] for x in results if x['version'] == version]) + print("{}/{}: {}".format(domain, version, ip_obs)) + ip_real = real.get(domain).get(version) + sys.stdout.flush() + if ip_obs == ip_real: + print('{}/{} is same!'.format(domain, version)) + else: + print("{}/{} diff on dns: real: {}, obs: {}".format(domain, version, ip_real, ip_obs)) + if set_dns(domain, version, ip_real): + print("update successful {}/{} -> {}".format(domain, version, ip_real)) + else: + print("update failed: {}/{} on dns is {}, but obs is {}".format( + domain, version, ip_real, ip_obs + )) + + +if __name__ == "__main__": + while True: + print("{}: Waking up".format(datetime.utcnow())) + sys.stdout.flush() + main() + print("{}: Sleeping for {}".format(datetime.utcnow(), SLEEP_SECONDS)) + sys.stdout.flush() + sleep(SLEEP_SECONDS) diff --git a/provision/auto_dns/dns_addresses.csv b/provision/auto_dns/dns_addresses.csv new file mode 100644 index 0000000..7560f43 --- /dev/null +++ b/provision/auto_dns/dns_addresses.csv @@ -0,0 +1,2 @@ +oc.pi2.dev,A +oc.pi2.dev,AAAA \ No newline at end of file diff --git a/provision/auto_dns/get_my_ip.py b/provision/auto_dns/get_my_ip.py new file mode 100644 index 0000000..3b53b61 --- /dev/null +++ b/provision/auto_dns/get_my_ip.py @@ -0,0 +1,6 @@ +import sys + + +if __name__ == "__main__": + domain = sys.argv[1] + diff --git a/provision/gunicorn.service b/provision/gunicorn.service deleted file mode 100644 index d671e41..0000000 --- a/provision/gunicorn.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=gunicorn service -Requires=network.target -After=network.target - -[Service] -User=pi2 -Type=simple -WorkingDirectory=/home/pi2/watch -ExecStart=/bin/bash -c 'export OSHIPKA_PATH=/home/pi2/oshipka; $OSHIPKA_PATH/oshipka.sh prod 5000' -Restart=on-failure - -[Install] -WantedBy=default.target diff --git a/provision/prod_mgmt.py b/provision/prod_mgmt.py new file mode 100644 index 0000000..182cce3 --- /dev/null +++ b/provision/prod_mgmt.py @@ -0,0 +1,56 @@ +import os + +from jinja2 import FileSystemLoader, Environment + +from oshipka.util.os import run_cmd +from provision.util import find_port, find_private_ipv4 + +USER = "pi2" +PARENT_DOMAIN = "pi2.dev" + + +oshipka_path = os.environ.get('OSHIPKA_PATH') +TEMPLATES_PATH = os.path.join(oshipka_path, "provision", "templates") +TMP_PATH = os.path.join(oshipka_path, "provision", "tmp") +os.makedirs(TMP_PATH, exist_ok=True) +env = Environment( + loader=FileSystemLoader(searchpath=TEMPLATES_PATH), +) + + +def prod_install(): + pwd = next(run_cmd('pwd')).strip() + + project_name = os.path.basename(pwd) + project_domain = "{}.{}".format(project_name, PARENT_DOMAIN) + port = find_port() + upstream_ip = find_private_ipv4() + + ctx = dict( + user=USER, + pwd=pwd, + oshipka_path=oshipka_path, + project_name=project_name, + project_domain=project_domain, + upstream_ip=upstream_ip, + port=port, + ) + + service_tmpl = env.get_template('gunicorn.service') + service_txt = service_tmpl.render(ctx) + with open(os.path.join(TMP_PATH, "{}.service".format(project_name)), 'w') as f: + f.write(service_txt) + + worker_service_tmpl = env.get_template('worker.service') + worker_service_txt = worker_service_tmpl.render(ctx) + with open(os.path.join(TMP_PATH, "{}_worker.service".format(project_name)), 'w') as f: + f.write(worker_service_txt) + + nginx_tmpl = env.get_template('nginx.conf') + nginx_txt = nginx_tmpl.render(ctx) + with open(os.path.join(TMP_PATH, "{}.conf".format(project_domain)), 'w') as f: + f.write(nginx_txt) + + +if __name__ == "__main__": + prod_install() diff --git a/provision/templates/gunicorn.service b/provision/templates/gunicorn.service new file mode 100644 index 0000000..68e016a --- /dev/null +++ b/provision/templates/gunicorn.service @@ -0,0 +1,14 @@ +[Unit] +Description=oshipka {{ project_name }} gunicorn service +Requires=network.target +After=network.target + +[Service] +User={{ user }} +Type=simple +WorkingDirectory={{ pwd }} +ExecStart=/bin/bash -c 'export OSHIPKA_PATH={{ oshipka_path }}; $OSHIPKA_PATH/oshipka.sh prod {{ port }}' +Restart=on-failure + +[Install] +WantedBy=default.target diff --git a/provision/nginx.conf b/provision/templates/nginx.conf similarity index 79% rename from provision/nginx.conf rename to provision/templates/nginx.conf index 1c8605e..cc56928 100644 --- a/provision/nginx.conf +++ b/provision/templates/nginx.conf @@ -2,7 +2,7 @@ server { listen 80; listen [::]:80; - server_name PROJECT_DOMAIN; + server_name {{ project_domain }}; # redirect all HTTP requests to HTTPS with a 301 Moved Permanently response. return 301 https://; @@ -12,15 +12,15 @@ server { listen 443 ssl http2; listen [::]:443 ssl http2; - server_name PROJECT_DOMAIN; + server_name {{ project_domain }}; server_tokens off; charset utf-8; - client_max_body_size 1g; + client_max_body_size {{ project_domain }}; # certs sent to the client in SERVER HELLO are concatenated in ssl_certificate - ssl_certificate /etc/letsencrypt/live/PROJECT_DOMAIN/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/PROJECT_DOMAIN/privkey.pem; + ssl_certificate /etc/letsencrypt/live/{{ project_domain }}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/{{ project_domain }}/privkey.pem; ssl_session_timeout 1d; ssl_session_cache shared:MozSSL:10m; # about 40000 sessions ssl_session_tickets off; @@ -44,10 +44,10 @@ server { # replace with the IP address of your resolver location / { - proxy_pass http://192.168.1.112:5001; + proxy_pass http://{{ upstream_ip }}:{{ port }}; proxy_redirect off; - proxy_set_header Host $host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/provision/templates/worker.service b/provision/templates/worker.service new file mode 100644 index 0000000..5c998c8 --- /dev/null +++ b/provision/templates/worker.service @@ -0,0 +1,14 @@ +[Unit] +Description=oshipka {{ project_name }} worker service +Requires=network.target +After=network.target + +[Service] +User={{ user }} +Type=simple +WorkingDirectory={{ pwd }} +ExecStart=/bin/bash -c 'export OSHIPKA_PATH={{ oshipka_path }}; $OSHIPKA_PATH/oshipka.sh worker' +Restart=on-failure + +[Install] +WantedBy=default.target diff --git a/provision/util.py b/provision/util.py new file mode 100644 index 0000000..86342ca --- /dev/null +++ b/provision/util.py @@ -0,0 +1,27 @@ +import re + +from oshipka.util.os import run_os_cmd + +PORT = 5000 + +ipv4_regex = re.compile('^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$') + + +def find_port(): + for i in range(16): + port = PORT + i + has_port = run_os_cmd("ss -tl | grep {}".format(port)) + if not has_port: + return port + + +def find_private_ipv4(): + private_ips = run_os_cmd("hostname -I").split() + for private_ip in private_ips: + private_ip = private_ip.strip() + if ipv4_regex.match(private_ip): + return private_ip + + +if __name__ == "__main__": + print(find_port()) diff --git a/requirements.txt b/requirements.txt index 295150a..f61728f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ bcrypt==3.1.7 blinker==1.4 cffi==1.14.0 click==7.1.1 +dns-lexicon==3.3.11 filelock==3.0.12 Flask==1.1.1 Flask-Babel==1.0.0