daniel
/
oshipka
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

automating provisioning

This commit is contained in:
Daniel Tsvetkov 2020-06-21 11:17:34 +02:00
parent 625fe533d5
commit 03c2addf8a
18 changed files with 417 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -3,4 +3,6 @@ venv
*.pyc
data/db.sqlite
__pycache__
oshipka.egg-info
oshipka.egg-info
provision/tmp
secrets.py

2
bootstrap/config.py
View File

@ -5,7 +5,7 @@ basepath = os.path.dirname(os.path.realpath(__file__))
DATA_DIR = os.path.join(basepath, "data")
STATIC_DATA_DIR = os.path.join(basepath, "data_static")
MEDIA_DIR = os.path.join(basepath, "media")
MEDIA_DIR = os.path.join(DATA_DIR, "media")
TASKS_DIR = os.path.join(DATA_DIR, "tasks")
TASKS_IN_DIR = os.path.join(TASKS_DIR, "in")

56
oshipka.sh
View File

@ -28,6 +28,7 @@ Usage $0 [ bootstrap | model | db_migrate | db_upgrade | db_populate | db_recrea
link Link dev oshipka
prod Run in prod
prod_install Install in prod
cert [DOMAIN] Install certificate
"
@ -69,7 +70,7 @@ install_cert() {
shift
PROJECT_DOMAIN=$1
sudo apt install certbot
certbot certonly --authenticator standalone --installer nginx --pre-hook "service nginx stop" --post-hook "service nginx start" --redirect --agree-tos --no-eff-email --email admin@app.com -d ${PROJECT_DOMAIN} --no-bootstrap
certbot certonly --authenticator standalone --installer nginx --pre-hook "service nginx stop" --post-hook "service nginx start" --redirect --agree-tos --no-eff-email --email danieltcv@gmail.com -d ${PROJECT_DOMAIN} --no-bootstrap
}
bootstrap() {
@ -121,6 +122,57 @@ run_in_prod() {
gunicorn -w 4 -b 0.0.0.0:${PORT} run:app
}
prod_install() {
shift
source venv/bin/activate
PROJECT_NAME=$(basename `pwd`)
echo "1/4 Generating service and config files..."
"${OSHIPKA_PATH}/venv/bin/python" "${OSHIPKA_PATH}/provision/prod_mgmt.py"
if [ -f "/etc/systemd/system/${PROJECT_NAME}.service" ]; then
echo "Service gunicorn for ${PROJECT_NAME} service exists."
systemctl status ${PROJECT_NAME}
else
echo "Installing '$PROJECT_NAME' gunicorn service"
sudo cp "${OSHIPKA_PATH}/provision/tmp/${PROJECT_NAME}.service" /etc/systemd/system/
sudo systemctl enable "${PROJECT_NAME}"
sudo systemctl start "${PROJECT_NAME}"
fi
echo "2/5 Installing '$PROJECT_NAME' worker service"
if [ -f "/etc/systemd/system/${PROJECT_NAME}_worker.service" ]; then
echo "Service worker for ${PROJECT_NAME} service exists."
systemctl status "${PROJECT_NAME}_worker"
else
sudo cp "${OSHIPKA_PATH}/provision/tmp/${PROJECT_NAME}_worker.service" /etc/systemd/system/
sudo systemctl enable "${PROJECT_NAME}_worker"
sudo systemctl start "${PROJECT_NAME}_worker"
fi
# TODO: Update DNS
NGINX_CONFIG_FILE=$(basename `find $OSHIPKA_PATH/provision/tmp -name *.conf`)
DOMAIN=$(basename -s .conf $NGINX_CONFIG_FILE)
echo "3/5 Installing '$DOMAIN' domain..."
"${OSHIPKA_PATH}/venv/bin/python" "${OSHIPKA_PATH}/audo_dns/set_domain.py" "$DOMAIN"
# TODO: Create certificate
echo "4/5 Installing '$PROJECT_NAME' certificate..."
install_cert $DOMAIN
echo "5/5 Installing '$PROJECT_NAME' nginx config..."
if [ -f "/etc/nginx/sites-available/${NGINX_CONFIG_FILE}" ]; then
echo "Nginx config for ${PROJECT_NAME} available."
if [ -f "/etc/nginx/sites-enabled/${NGINX_CONFIG_FILE}" ]; then
echo "Nginx config for ${PROJECT_NAME} enabled."
else
echo "Nginx config for ${PROJECT_NAME} NOT enabled."
fi
else
echo "Installing nginx config for ${PROJECT_NAME} -> enabling + available."
sudo cp "${OSHIPKA_PATH}/provision/tmp/${NGINX_CONFIG_FILE}" /etc/nginx/sites-enabled/
sudo ln -s "/etc/nginx/sites-available/${NGINX_CONFIG_FILE}" "/etc/nginx/sites-enabled/${NGINX_CONFIG_FILE}"
fi
}
model() {
shift
source venv/bin/activate
@ -202,6 +254,8 @@ command_main() {
;;
prod) run_in_prod "$@"
;;
prod_install) prod_install "$@"
;;
cert) install_cert "$@"
;;
*) >&2 echo -e "${HELP}"

14
oshipka/util/os.py
View File

@ -1,13 +1,21 @@
import os
import shlex
import subprocess
def run_os_cmd(command):
output = os.popen(command).read()
return output
def run_cmd(command):
process = subprocess.Popen(command.split(),
process = subprocess.Popen(shlex.split(command),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
return '\n'.join([str(line) for line in iter(process.stdout.readline, '')])
for line in iter(process.stdout.readline, ''):
yield line
def print_cmd(command):
@ -16,4 +24,4 @@ def print_cmd(command):
if __name__ == "__main__":
print_cmd("env")
print_cmd("env")

35
oshipka/util/process.py
View File

@ -1,5 +1,36 @@
import traceback
from time import time
from time import time, sleep
def process_exp_backoff(func, func_args=None, func_kwargs=None, func_exc_classes=None,
max_attempts=3, initial_sleep_time=1, sleep_exp=2, max_sleep_time=0, timeout=0):
func_args = list() if func_args is None else func_args
func_kwargs = dict() if func_kwargs is None else func_kwargs
func_exc_classes = Exception if func_exc_classes is None else func_exc_classes
attempt, start_time, sleep_time = 1, time(), initial_sleep_time
while True:
try:
print("EXP_BACKOFF: Attempt {}/{}".format(attempt, max_attempts))
return func(*func_args, **func_kwargs)
except func_exc_classes as e:
print("EXP_BACKOFF: Received error at attempt: {}/{}".format(attempt, max_attempts))
traceback.print_exc()
print(e)
total_time = time() - start_time
if (max_sleep_time and sleep_time >= max_sleep_time) or \
(max_attempts and attempt >= max_attempts) or \
(timeout and timeout >= total_time):
raise Exception("EXP_BACKOFF: Giving up: \n"
"- sleep_time: {}/{}\n"
"- attempt: {}/{}\n"
"- total_time: {}/{}\n".format(
sleep_time, max_sleep_time,
attempt, max_attempts,
total_time, timeout))
sleep_time *= sleep_exp
print("EXP_BACKOFF: Sleeping {}...".format(sleep_time))
sleep(sleep_time)
attempt += 1
def process_iterable(iterable: set, process_func, processed: set = None, errors: set = None,
@ -9,7 +40,7 @@ def process_iterable(iterable: set, process_func, processed: set = None, errors:
f_args = list() if not f_args else f_args
f_kwargs = dict() if not f_kwargs else f_kwargs
to_process = iterable# - processed - errors
to_process = iterable # - processed - errors
tot_iterables = len(to_process)
tot_start = time()

0
provision/__init__.py Normal file
View File

0
provision/auto_dns/__init__.py Normal file
View File

15
provision/auto_dns/auto_dns.service Normal file
View File

@ -0,0 +1,15 @@
[Unit]
Description=auto dns
Requires=network.target
After=network.target
[Service]
Type=simple
User={{ user }}
Type=simple
WorkingDirectory={{ oshipka_path }}
ExecStart=/bin/bash -c '{{ oshipka_path }}/venv/bin/python {{ oshipka_path }}/provision/auto_dns/check.py'
Restart=on-failure
[Install]
WantedBy=default.target

172
provision/auto_dns/check.py Normal file
View File

@ -0,0 +1,172 @@
import csv
import ipaddress
import threading
from collections import defaultdict
from datetime import datetime
from time import sleep
import sys
import requests
from lexicon.client import Client
from lexicon.config import ConfigResolver
from oshipka.util.process import process_exp_backoff
from secrets import HOVER_USERNAME, HOVER_PASSWORD
SLEEP_SECONDS = 60 * 30 # 30 min
ipv4_sites = [
"http://www.icanhazip.com",
"http://ipecho.net/plain",
"http://checkip.amazonaws.com",
"http://ipinfo.io/ip",
"http://ifconfig.me"
]
ipv6_sites = [
"http://bot.whatismyipaddress.com",
"https://diagnostic.opendns.com/myip",
"https://ifconfig.co/ip",
]
dns_to_check = []
with open('dns_addresses.csv') as csvf:
csvreader = csv.reader(csvf)
for row in csvreader:
dns_to_check.append(row)
ip_checkers = ipv4_sites + ipv6_sites
results = []
real = defaultdict(dict)
def test_ip(ip):
try:
ipaddress_info = ipaddress.ip_address(ip)
except ValueError as e:
print(e)
return None
if ipaddress_info.version == 4:
version = 'A'
elif ipaddress_info.version == 6:
version = 'AAAA'
else:
return None
return {
"addr": ipaddress_info.exploded,
"version": version,
}
def fetch_ip(url):
try:
resp = requests.get(url)
ip = resp.text.strip()
ip_info = test_ip(ip)
if ip_info:
results.append(ip_info)
except Exception as e:
print("Error fetching {}: {}".format(url, e))
tso, tsd = [], []
def check_observed():
for url in ip_checkers:
t = threading.Thread(target=fetch_ip, args=(url,))
tso.append(t)
t.start()
def check_dns():
for domain, ipv in dns_to_check:
t = threading.Thread(target=get_dns, args=(domain, ipv,))
tsd.append(t)
t.start()
def get_hover_results(domain, ipv):
dns_config = ConfigResolver()
dns_config.with_dict({
'provider_name': 'hover',
'action': 'list',
'type': ipv,
'domain': domain,
'hover': {
'auth_username': HOVER_USERNAME,
'auth_password': HOVER_PASSWORD,
}
})
client = Client(dns_config)
return client.execute()
def set_hover_results(domain, ipv, new):
dns_config = ConfigResolver()
dns_config.with_dict({
'provider_name': 'hover',
'action': 'update',
'type': ipv,
'domain': domain,
'name': domain,
'content': new,
'hover': {
'auth_username': HOVER_USERNAME,
'auth_password': HOVER_PASSWORD,
}
})
client = Client(dns_config)
return client.execute()
def get_dns(domain, ipv):
results = process_exp_backoff(get_hover_results, func_args=[domain, ipv])
res = [x.get('content') for x in results if x.get('name') == domain]
if len(res) == 1:
print("Real {}: {}".format(ipv, res[0]))
real[domain][ipv] = res[0]
def set_dns(domain, ipv, new):
return process_exp_backoff(set_hover_results, func_args=[domain, ipv, new])
def get_uniq(_list):
if len(_list) > 1 and len(set(_list)) == 1:
return _list[0]
def main():
check_dns()
check_observed()
for t in tso:
t.join()
for t in tsd:
t.join()
for domain, version in dns_to_check:
ip_obs = get_uniq([x['addr'] for x in results if x['version'] == version])
print("{}/{}: {}".format(domain, version, ip_obs))
ip_real = real.get(domain).get(version)
sys.stdout.flush()
if ip_obs == ip_real:
print('{}/{} is same!'.format(domain, version))
else:
print("{}/{} diff on dns: real: {}, obs: {}".format(domain, version, ip_real, ip_obs))
if set_dns(domain, version, ip_real):
print("update successful {}/{} -> {}".format(domain, version, ip_real))
else:
print("update failed: {}/{} on dns is {}, but obs is {}".format(
domain, version, ip_real, ip_obs
))
if __name__ == "__main__":
while True:
print("{}: Waking up".format(datetime.utcnow()))
sys.stdout.flush()
main()
print("{}: Sleeping for {}".format(datetime.utcnow(), SLEEP_SECONDS))
sys.stdout.flush()
sleep(SLEEP_SECONDS)

2
provision/auto_dns/dns_addresses.csv Normal file
View File

@ -0,0 +1,2 @@
oc.pi2.dev,A
oc.pi2.dev,AAAA
1 oc.pi2.dev A
2 oc.pi2.dev AAAA

6
provision/auto_dns/get_my_ip.py Normal file
View File

@ -0,0 +1,6 @@
import sys
if __name__ == "__main__":
domain = sys.argv[1]

14
provision/gunicorn.service
View File

@ -1,14 +0,0 @@
[Unit]
Description=gunicorn service
Requires=network.target
After=network.target
[Service]
User=pi2
Type=simple
WorkingDirectory=/home/pi2/watch
ExecStart=/bin/bash -c 'export OSHIPKA_PATH=/home/pi2/oshipka; $OSHIPKA_PATH/oshipka.sh prod 5000'
Restart=on-failure
[Install]
WantedBy=default.target

56
provision/prod_mgmt.py Normal file
View File

@ -0,0 +1,56 @@
import os
from jinja2 import FileSystemLoader, Environment
from oshipka.util.os import run_cmd
from provision.util import find_port, find_private_ipv4
USER = "pi2"
PARENT_DOMAIN = "pi2.dev"
oshipka_path = os.environ.get('OSHIPKA_PATH')
TEMPLATES_PATH = os.path.join(oshipka_path, "provision", "templates")
TMP_PATH = os.path.join(oshipka_path, "provision", "tmp")
os.makedirs(TMP_PATH, exist_ok=True)
env = Environment(
loader=FileSystemLoader(searchpath=TEMPLATES_PATH),
)
def prod_install():
pwd = next(run_cmd('pwd')).strip()
project_name = os.path.basename(pwd)
project_domain = "{}.{}".format(project_name, PARENT_DOMAIN)
port = find_port()
upstream_ip = find_private_ipv4()
ctx = dict(
user=USER,
pwd=pwd,
oshipka_path=oshipka_path,
project_name=project_name,
project_domain=project_domain,
upstream_ip=upstream_ip,
port=port,
)
service_tmpl = env.get_template('gunicorn.service')
service_txt = service_tmpl.render(ctx)
with open(os.path.join(TMP_PATH, "{}.service".format(project_name)), 'w') as f:
f.write(service_txt)
worker_service_tmpl = env.get_template('worker.service')
worker_service_txt = worker_service_tmpl.render(ctx)
with open(os.path.join(TMP_PATH, "{}_worker.service".format(project_name)), 'w') as f:
f.write(worker_service_txt)
nginx_tmpl = env.get_template('nginx.conf')
nginx_txt = nginx_tmpl.render(ctx)
with open(os.path.join(TMP_PATH, "{}.conf".format(project_domain)), 'w') as f:
f.write(nginx_txt)
if __name__ == "__main__":
prod_install()

14
provision/templates/gunicorn.service Normal file
View File

@ -0,0 +1,14 @@
[Unit]
Description=oshipka {{ project_name }} gunicorn service
Requires=network.target
After=network.target
[Service]
User={{ user }}
Type=simple
WorkingDirectory={{ pwd }}
ExecStart=/bin/bash -c 'export OSHIPKA_PATH={{ oshipka_path }}; $OSHIPKA_PATH/oshipka.sh prod {{ port }}'
Restart=on-failure
[Install]
WantedBy=default.target

14
provision/nginx.conf → provision/templates/nginx.conf
View File

@ -2,7 +2,7 @@ server {
listen 80;
listen [::]:80;
server_name PROJECT_DOMAIN;
server_name {{ project_domain }};
# redirect all HTTP requests to HTTPS with a 301 Moved Permanently response.
return 301 https://;
@ -12,15 +12,15 @@ server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name PROJECT_DOMAIN;
server_name {{ project_domain }};
server_tokens off;
charset utf-8;
client_max_body_size 1g;
client_max_body_size {{ project_domain }};
# certs sent to the client in SERVER HELLO are concatenated in ssl_certificate
ssl_certificate /etc/letsencrypt/live/PROJECT_DOMAIN/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/PROJECT_DOMAIN/privkey.pem;
ssl_certificate /etc/letsencrypt/live/{{ project_domain }}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/{{ project_domain }}/privkey.pem;
ssl_session_timeout 1d;
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
ssl_session_tickets off;
@ -44,10 +44,10 @@ server {
# replace with the IP address of your resolver
location / {
proxy_pass http://192.168.1.112:5001;
proxy_pass http://{{ upstream_ip }}:{{ port }};
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

14
provision/templates/worker.service Normal file
View File

@ -0,0 +1,14 @@
[Unit]
Description=oshipka {{ project_name }} worker service
Requires=network.target
After=network.target
[Service]
User={{ user }}
Type=simple
WorkingDirectory={{ pwd }}
ExecStart=/bin/bash -c 'export OSHIPKA_PATH={{ oshipka_path }}; $OSHIPKA_PATH/oshipka.sh worker'
Restart=on-failure
[Install]
WantedBy=default.target

27
provision/util.py Normal file
View File

@ -0,0 +1,27 @@
import re
from oshipka.util.os import run_os_cmd
PORT = 5000
ipv4_regex = re.compile('^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$')
def find_port():
for i in range(16):
port = PORT + i
has_port = run_os_cmd("ss -tl | grep {}".format(port))
if not has_port:
return port
def find_private_ipv4():
private_ips = run_os_cmd("hostname -I").split()
for private_ip in private_ips:
private_ip = private_ip.strip()
if ipv4_regex.match(private_ip):
return private_ip
if __name__ == "__main__":
print(find_port())

1
requirements.txt
View File

@ -4,6 +4,7 @@ bcrypt==3.1.7
blinker==1.4
cffi==1.14.0
click==7.1.1
dns-lexicon==3.3.11
filelock==3.0.12
Flask==1.1.1
Flask-Babel==1.0.0