From ed841fd03319e8a5927e909f5577b4530af16b23 Mon Sep 17 00:00:00 2001 From: Daniel Tsvetkov Date: Sun, 21 Jun 2020 17:45:33 +0200 Subject: [PATCH] testing setting up a site end-2-end --- .gitignore | 2 +- oshipka.sh | 53 +++++++++++++++++-------- provision/auto_dns/check.py | 43 ++++++++++---------- provision/auto_dns/get_my_ip.py | 6 --- provision/auto_dns/set_domain_ipv4.py | 40 +++++++++++++++++++ provision/prod_mgmt.py | 26 ++++++------ provision/templates/nginx.conf | 2 +- provision/templates/nginx_insecure.conf | 15 +++++++ 8 files changed, 128 insertions(+), 59 deletions(-) delete mode 100644 provision/auto_dns/get_my_ip.py create mode 100644 provision/auto_dns/set_domain_ipv4.py create mode 100644 provision/templates/nginx_insecure.conf diff --git a/.gitignore b/.gitignore index 69df644..c20f20a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ data/db.sqlite __pycache__ oshipka.egg-info provision/tmp -secrets.py \ No newline at end of file +provision/auto_dns/sensitive.py \ No newline at end of file diff --git a/oshipka.sh b/oshipka.sh index 777ce15..a8b77b7 100755 --- a/oshipka.sh +++ b/oshipka.sh @@ -64,13 +64,9 @@ init() { } install_cert() { - #[ ! -f /tmp/certbot-auto ] && wget -O /tmp/certbot-auto https://dl.eff.org/certbot-auto -#chmod +x /tmp/certbot-auto -#certbot=/tmp/certbot-auto - shift PROJECT_DOMAIN=$1 sudo apt install certbot - certbot certonly --authenticator standalone --installer nginx --pre-hook "service nginx stop" --post-hook "service nginx start" --redirect --agree-tos --no-eff-email --email danieltcv@gmail.com -d ${PROJECT_DOMAIN} --no-bootstrap + sudo certbot certonly --dry-run --authenticator standalone --installer nginx --pre-hook "service nginx stop" --post-hook "service nginx start" --redirect --agree-tos --no-eff-email --email danieltcv@gmail.com -d ${PROJECT_DOMAIN} --no-bootstrap } bootstrap() { @@ -123,11 +119,12 @@ run_in_prod() { } prod_install() { + set -e shift source venv/bin/activate PROJECT_NAME=$(basename `pwd`) - echo "1/4 Generating service and config files..." - "${OSHIPKA_PATH}/venv/bin/python" "${OSHIPKA_PATH}/provision/prod_mgmt.py" + echo "1/6 Generating service and config files..." + python "${OSHIPKA_PATH}/provision/prod_mgmt.py" if [ -f "/etc/systemd/system/${PROJECT_NAME}.service" ]; then echo "Service gunicorn for ${PROJECT_NAME} service exists." systemctl status ${PROJECT_NAME} @@ -138,7 +135,7 @@ prod_install() { sudo systemctl start "${PROJECT_NAME}" fi - echo "2/5 Installing '$PROJECT_NAME' worker service" + echo "2/6 Installing '$PROJECT_NAME' worker service" if [ -f "/etc/systemd/system/${PROJECT_NAME}_worker.service" ]; then echo "Service worker for ${PROJECT_NAME} service exists." systemctl status "${PROJECT_NAME}_worker" @@ -148,17 +145,38 @@ prod_install() { sudo systemctl start "${PROJECT_NAME}_worker" fi - # TODO: Update DNS NGINX_CONFIG_FILE=$(basename `find $OSHIPKA_PATH/provision/tmp -name *.conf`) DOMAIN=$(basename -s .conf $NGINX_CONFIG_FILE) - echo "3/5 Installing '$DOMAIN' domain..." - "${OSHIPKA_PATH}/venv/bin/python" "${OSHIPKA_PATH}/audo_dns/set_domain.py" "$DOMAIN" + echo "3/6 Installing '$DOMAIN' domain..." + python "${OSHIPKA_PATH}/provision/auto_dns/set_domain_ipv4.py" "$DOMAIN" - # TODO: Create certificate - echo "4/5 Installing '$PROJECT_NAME' certificate..." + sudo apt install nginx + sudo systemctl start nginx + echo "Enabling firewall rule for 192.168.1.1 -> 80/tcp..." + sudo ufw allow proto tcp from 192.168.1.1 to any port 80 + echo "4/6 Installing '$PROJECT_NAME' insecure nginx config..." + if [ -f "/etc/nginx/sites-available/${DOMAIN}_insecure.conf" ]; then + echo "Insecure Nginx config for ${PROJECT_NAME} available." + if [ -f "/etc/nginx/sites-enabled/${DOMAIN}_insecure.conf" ]; then + echo "Nginx config for ${PROJECT_NAME} enabled." + else + echo "Nginx config for ${PROJECT_NAME} NOT enabled." + fi + else + echo "Installing insecure nginx config for ${PROJECT_NAME} -> enabling + available." + sudo cp "${OSHIPKA_PATH}/provision/tmp/${DOMAIN}_insecure.conf" /etc/nginx/sites-available/ + sudo ln -s "/etc/nginx/sites-available/${DOMAIN}_insecure.conf" "/etc/nginx/sites-enabled/${DOMAIN}_insecure.conf" + sudo systemctl reload nginx + fi + + echo "5/6 Installing '$PROJECT_NAME' certificate..." install_cert $DOMAIN - echo "5/5 Installing '$PROJECT_NAME' nginx config..." + echo "6/6 Installing '$PROJECT_NAME' secure nginx config..." + echo "Enabling firewall rule for 192.168.1.1 -> 443/tcp..." + sudo ufw allow proto tcp from 192.168.1.1 to any port 443 + echo "Removing '$PROJECT_NAME' insecure nginx config..." + sudo rm "/etc/nginx/sites-available/${DOMAIN}_insecure.conf" "/etc/nginx/sites-enabled/${DOMAIN}_insecure.conf" if [ -f "/etc/nginx/sites-available/${NGINX_CONFIG_FILE}" ]; then echo "Nginx config for ${PROJECT_NAME} available." if [ -f "/etc/nginx/sites-enabled/${NGINX_CONFIG_FILE}" ]; then @@ -166,10 +184,11 @@ prod_install() { else echo "Nginx config for ${PROJECT_NAME} NOT enabled." fi - else + else echo "Installing nginx config for ${PROJECT_NAME} -> enabling + available." - sudo cp "${OSHIPKA_PATH}/provision/tmp/${NGINX_CONFIG_FILE}" /etc/nginx/sites-enabled/ + sudo cp "${OSHIPKA_PATH}/provision/tmp/${NGINX_CONFIG_FILE}" /etc/nginx/sites-available/ sudo ln -s "/etc/nginx/sites-available/${NGINX_CONFIG_FILE}" "/etc/nginx/sites-enabled/${NGINX_CONFIG_FILE}" + sudo systemctl reload nginx fi } @@ -256,7 +275,7 @@ command_main() { ;; prod_install) prod_install "$@" ;; - cert) install_cert "$@" + cert) shift && install_cert "$@" ;; *) >&2 echo -e "${HELP}" return 1 diff --git a/provision/auto_dns/check.py b/provision/auto_dns/check.py index a96d584..4f39021 100644 --- a/provision/auto_dns/check.py +++ b/provision/auto_dns/check.py @@ -11,10 +11,12 @@ from lexicon.client import Client from lexicon.config import ConfigResolver from oshipka.util.process import process_exp_backoff -from secrets import HOVER_USERNAME, HOVER_PASSWORD +from sensitive import CLOUDFLARE_USERNAME, CLOUDFLARE_AUTH_KEY SLEEP_SECONDS = 60 * 30 # 30 min +DEFAULT_TTL = 5 * 60 + ipv4_sites = [ "http://www.icanhazip.com", "http://ipecho.net/plain", @@ -24,19 +26,13 @@ ipv4_sites = [ ] ipv6_sites = [ - "http://bot.whatismyipaddress.com", "https://diagnostic.opendns.com/myip", "https://ifconfig.co/ip", ] -dns_to_check = [] -with open('dns_addresses.csv') as csvf: - csvreader = csv.reader(csvf) - for row in csvreader: - dns_to_check.append(row) - ip_checkers = ipv4_sites + ipv6_sites +dns_to_check = [] results = [] real = defaultdict(dict) @@ -87,34 +83,35 @@ def check_dns(): t.start() -def get_hover_results(domain, ipv): +def get_resolver_results(domain, ipv): dns_config = ConfigResolver() dns_config.with_dict({ - 'provider_name': 'hover', + 'provider_name': 'cloudflare', 'action': 'list', 'type': ipv, 'domain': domain, - 'hover': { - 'auth_username': HOVER_USERNAME, - 'auth_password': HOVER_PASSWORD, + 'cloudflare': { + 'auth_username': CLOUDFLARE_USERNAME, + 'auth_token': CLOUDFLARE_AUTH_KEY, } }) client = Client(dns_config) return client.execute() -def set_hover_results(domain, ipv, new): +def set_resolver_results(domain, ipv, new): dns_config = ConfigResolver() dns_config.with_dict({ - 'provider_name': 'hover', - 'action': 'update', + 'provider_name': 'cloudflare', + 'action': 'create', 'type': ipv, 'domain': domain, 'name': domain, 'content': new, - 'hover': { - 'auth_username': HOVER_USERNAME, - 'auth_password': HOVER_PASSWORD, + 'ttl': DEFAULT_TTL, + 'cloudflare': { + 'auth_username': CLOUDFLARE_USERNAME, + 'auth_token': CLOUDFLARE_AUTH_KEY, } }) client = Client(dns_config) @@ -122,7 +119,7 @@ def set_hover_results(domain, ipv, new): def get_dns(domain, ipv): - results = process_exp_backoff(get_hover_results, func_args=[domain, ipv]) + results = process_exp_backoff(get_resolver_results, func_args=[domain, ipv]) res = [x.get('content') for x in results if x.get('name') == domain] if len(res) == 1: print("Real {}: {}".format(ipv, res[0])) @@ -130,7 +127,7 @@ def get_dns(domain, ipv): def set_dns(domain, ipv, new): - return process_exp_backoff(set_hover_results, func_args=[domain, ipv, new]) + return process_exp_backoff(set_resolver_results, func_args=[domain, ipv, new]) def get_uniq(_list): @@ -139,6 +136,10 @@ def get_uniq(_list): def main(): + with open('dns_addresses.csv') as csvf: + csvreader = csv.reader(csvf) + for row in csvreader: + dns_to_check.append(row) check_dns() check_observed() for t in tso: diff --git a/provision/auto_dns/get_my_ip.py b/provision/auto_dns/get_my_ip.py deleted file mode 100644 index 3b53b61..0000000 --- a/provision/auto_dns/get_my_ip.py +++ /dev/null @@ -1,6 +0,0 @@ -import sys - - -if __name__ == "__main__": - domain = sys.argv[1] - diff --git a/provision/auto_dns/set_domain_ipv4.py b/provision/auto_dns/set_domain_ipv4.py new file mode 100644 index 0000000..896985d --- /dev/null +++ b/provision/auto_dns/set_domain_ipv4.py @@ -0,0 +1,40 @@ +import random +import sys +from time import sleep + +import requests + +from oshipka.util.os import run_os_cmd +from oshipka.util.process import process_exp_backoff +from provision.auto_dns.check import ipv4_sites, set_resolver_results, DEFAULT_TTL + + +def verify_dns_domain(domain, ipv4): + ip_on_dns = run_os_cmd("dig +short {}".format(domain)) + if not ip_on_dns: + raise Exception("Record for {} doesn't exist".format(domain)) + ip_on_dns = ip_on_dns.strip() + assert ip_on_dns == ipv4 + + +def set_domain_ipv4(domain): + url = random.choice(ipv4_sites) + resp = requests.get(url) + ipv4 = resp.text.strip() + print('Got IP: {}'.format(ipv4)) + try: + verify_dns_domain(domain, ipv4) + print("DNS already set up for {}".format(domain)) + except Exception: + process_exp_backoff(set_resolver_results, func_args=[domain, 'A', ipv4]) + print("Sleeping 5 sec...") + sleep(5) + print("Checking that {} is set".format(domain)) + process_exp_backoff(verify_dns_domain, func_args=[domain, ipv4], max_attempts=0, max_sleep_time=DEFAULT_TTL) + + +if __name__ == "__main__": + domain = sys.argv[1] + if not domain: + print('domain is required first argument') + set_domain_ipv4(domain) diff --git a/provision/prod_mgmt.py b/provision/prod_mgmt.py index 182cce3..3a5fb00 100644 --- a/provision/prod_mgmt.py +++ b/provision/prod_mgmt.py @@ -7,6 +7,7 @@ from provision.util import find_port, find_private_ipv4 USER = "pi2" PARENT_DOMAIN = "pi2.dev" +MAX_UPLOAD_SIZE = "10m" oshipka_path = os.environ.get('OSHIPKA_PATH') @@ -33,23 +34,22 @@ def prod_install(): project_name=project_name, project_domain=project_domain, upstream_ip=upstream_ip, + max_upload_size=MAX_UPLOAD_SIZE, port=port, ) - service_tmpl = env.get_template('gunicorn.service') - service_txt = service_tmpl.render(ctx) - with open(os.path.join(TMP_PATH, "{}.service".format(project_name)), 'w') as f: - f.write(service_txt) + tmpl_fname = [ + ('gunicorn.service', "{}.service".format(project_name)), + ('worker.service', "{}_worker.service".format(project_name)), + ('nginx_insecure.conf', "{}_insecure.conf".format(project_domain)), + ('nginx.conf', "{}.conf".format(project_domain)), + ] - worker_service_tmpl = env.get_template('worker.service') - worker_service_txt = worker_service_tmpl.render(ctx) - with open(os.path.join(TMP_PATH, "{}_worker.service".format(project_name)), 'w') as f: - f.write(worker_service_txt) - - nginx_tmpl = env.get_template('nginx.conf') - nginx_txt = nginx_tmpl.render(ctx) - with open(os.path.join(TMP_PATH, "{}.conf".format(project_domain)), 'w') as f: - f.write(nginx_txt) + for tmpl, fname in tmpl_fname: + actual_tmpl = env.get_template(tmpl) + rendered_tmpl = actual_tmpl.render(ctx) + with open(os.path.join(TMP_PATH, fname), 'w') as f: + f.write(rendered_tmpl) if __name__ == "__main__": diff --git a/provision/templates/nginx.conf b/provision/templates/nginx.conf index cc56928..b22dc98 100644 --- a/provision/templates/nginx.conf +++ b/provision/templates/nginx.conf @@ -16,7 +16,7 @@ server { server_tokens off; charset utf-8; - client_max_body_size {{ project_domain }}; + client_max_body_size {{ max_upload_size }}; # certs sent to the client in SERVER HELLO are concatenated in ssl_certificate ssl_certificate /etc/letsencrypt/live/{{ project_domain }}/fullchain.pem; diff --git a/provision/templates/nginx_insecure.conf b/provision/templates/nginx_insecure.conf new file mode 100644 index 0000000..af0cdb4 --- /dev/null +++ b/provision/templates/nginx_insecure.conf @@ -0,0 +1,15 @@ +server { + listen 80; + listen [::]:80; + + server_name {{ project_domain }}; + + location / { + proxy_pass http://{{ upstream_ip }}:{{ port }}; + proxy_redirect off; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +}