testing setting up a site end-2-end

.gitignore

@@ -5,4 +5,4 @@ data/db.sqlite
 __pycache__
 oshipka.egg-info
 provision/tmp
-secrets.py
+provision/auto_dns/sensitive.py

oshipka.sh

@@ -64,13 +64,9 @@ init() {
 }
 
 install_cert() {
-  #[ ! -f /tmp/certbot-auto ] && wget -O /tmp/certbot-auto https://dl.eff.org/certbot-auto
-#chmod +x /tmp/certbot-auto
-#certbot=/tmp/certbot-auto
-  shift
   PROJECT_DOMAIN=$1
   sudo apt install certbot
-  certbot certonly --authenticator standalone --installer nginx --pre-hook "service nginx stop" --post-hook "service nginx start" --redirect --agree-tos --no-eff-email --email danieltcv@gmail.com -d ${PROJECT_DOMAIN} --no-bootstrap
+  sudo certbot certonly --dry-run --authenticator standalone --installer nginx --pre-hook "service nginx stop" --post-hook "service nginx start" --redirect --agree-tos --no-eff-email --email danieltcv@gmail.com -d ${PROJECT_DOMAIN} --no-bootstrap
 }
 
 bootstrap() {
@@ -123,11 +119,12 @@ run_in_prod() {
 }
 
 prod_install() {
+   set -e
    shift
    source venv/bin/activate
    PROJECT_NAME=$(basename `pwd`)
-   echo "1/4 Generating service and config files..."
+   echo "1/6 Generating service and config files..."
-   "${OSHIPKA_PATH}/venv/bin/python" "${OSHIPKA_PATH}/provision/prod_mgmt.py"
+   python "${OSHIPKA_PATH}/provision/prod_mgmt.py"
    if [ -f "/etc/systemd/system/${PROJECT_NAME}.service" ]; then
      echo "Service gunicorn for ${PROJECT_NAME} service exists."
      systemctl status ${PROJECT_NAME}
@@ -138,7 +135,7 @@ prod_install() {
      sudo systemctl start "${PROJECT_NAME}"
    fi
 
-   echo "2/5 Installing '$PROJECT_NAME' worker service"
+   echo "2/6 Installing '$PROJECT_NAME' worker service"
    if [ -f "/etc/systemd/system/${PROJECT_NAME}_worker.service" ]; then
      echo "Service worker for ${PROJECT_NAME} service exists."
      systemctl status "${PROJECT_NAME}_worker"
@@ -148,17 +145,38 @@ prod_install() {
      sudo systemctl start "${PROJECT_NAME}_worker"
    fi
 
-   # TODO: Update DNS
    NGINX_CONFIG_FILE=$(basename `find $OSHIPKA_PATH/provision/tmp -name *.conf`)
    DOMAIN=$(basename -s .conf $NGINX_CONFIG_FILE)
-   echo "3/5 Installing '$DOMAIN' domain..."
+   echo "3/6 Installing '$DOMAIN' domain..."
-   "${OSHIPKA_PATH}/venv/bin/python" "${OSHIPKA_PATH}/audo_dns/set_domain.py" "$DOMAIN"
+   python "${OSHIPKA_PATH}/provision/auto_dns/set_domain_ipv4.py" "$DOMAIN"
 
-   # TODO: Create certificate
+   sudo apt install nginx
-   echo "4/5 Installing '$PROJECT_NAME' certificate..."
+   sudo systemctl start nginx
+   echo "Enabling firewall rule for 192.168.1.1 -> 80/tcp..."
+   sudo ufw allow proto tcp from 192.168.1.1 to any port 80
+   echo "4/6 Installing '$PROJECT_NAME' insecure nginx config..."
+   if [ -f "/etc/nginx/sites-available/${DOMAIN}_insecure.conf" ]; then
+     echo "Insecure Nginx config for ${PROJECT_NAME} available."
+     if [ -f "/etc/nginx/sites-enabled/${DOMAIN}_insecure.conf" ]; then
+       echo "Nginx config for ${PROJECT_NAME} enabled."
+     else
+       echo "Nginx config for ${PROJECT_NAME} NOT enabled."
+     fi
+   else
+       echo "Installing insecure nginx config for ${PROJECT_NAME} -> enabling + available."
+       sudo cp "${OSHIPKA_PATH}/provision/tmp/${DOMAIN}_insecure.conf" /etc/nginx/sites-available/
+       sudo ln -s "/etc/nginx/sites-available/${DOMAIN}_insecure.conf" "/etc/nginx/sites-enabled/${DOMAIN}_insecure.conf"
+       sudo systemctl reload nginx
+   fi
+
+   echo "5/6 Installing '$PROJECT_NAME' certificate..."
    install_cert $DOMAIN
 
-   echo "5/5 Installing '$PROJECT_NAME' nginx config..."
+   echo "6/6 Installing '$PROJECT_NAME' secure nginx config..."
+   echo "Enabling firewall rule for 192.168.1.1 -> 443/tcp..."
+   sudo ufw allow proto tcp from 192.168.1.1 to any port 443
+   echo "Removing '$PROJECT_NAME' insecure nginx config..."
+   sudo rm "/etc/nginx/sites-available/${DOMAIN}_insecure.conf" "/etc/nginx/sites-enabled/${DOMAIN}_insecure.conf"
    if [ -f "/etc/nginx/sites-available/${NGINX_CONFIG_FILE}" ]; then
      echo "Nginx config for ${PROJECT_NAME} available."
      if [ -f "/etc/nginx/sites-enabled/${NGINX_CONFIG_FILE}" ]; then
@@ -166,10 +184,11 @@ prod_install() {
      else
        echo "Nginx config for ${PROJECT_NAME} NOT enabled."
      fi
-     else
+   else
        echo "Installing nginx config for ${PROJECT_NAME} -> enabling + available."
-       sudo cp "${OSHIPKA_PATH}/provision/tmp/${NGINX_CONFIG_FILE}" /etc/nginx/sites-enabled/
+       sudo cp "${OSHIPKA_PATH}/provision/tmp/${NGINX_CONFIG_FILE}" /etc/nginx/sites-available/
        sudo ln -s "/etc/nginx/sites-available/${NGINX_CONFIG_FILE}" "/etc/nginx/sites-enabled/${NGINX_CONFIG_FILE}"
+       sudo systemctl reload nginx
    fi
 }
 
@@ -256,7 +275,7 @@ command_main() {
         ;;
         prod_install) prod_install "$@"
         ;;
-        cert) install_cert "$@"
+        cert) shift && install_cert "$@"
         ;;
         *) >&2 echo -e "${HELP}"
         return 1

provision/auto_dns/check.py

@@ -11,10 +11,12 @@ from lexicon.client import Client
 from lexicon.config import ConfigResolver
 
 from oshipka.util.process import process_exp_backoff
-from secrets import HOVER_USERNAME, HOVER_PASSWORD
+from sensitive import CLOUDFLARE_USERNAME, CLOUDFLARE_AUTH_KEY
 
 SLEEP_SECONDS = 60 * 30  # 30 min
 
+DEFAULT_TTL = 5 * 60
+
 ipv4_sites = [
     "http://www.icanhazip.com",
     "http://ipecho.net/plain",
@@ -24,19 +26,13 @@ ipv4_sites = [
 ]
 
 ipv6_sites = [
-    "http://bot.whatismyipaddress.com",
     "https://diagnostic.opendns.com/myip",
     "https://ifconfig.co/ip",
 ]
 
-dns_to_check = []
-with open('dns_addresses.csv') as csvf:
-    csvreader = csv.reader(csvf)
-    for row in csvreader:
-        dns_to_check.append(row)
-
 ip_checkers = ipv4_sites + ipv6_sites
 
+dns_to_check = []
 results = []
 real = defaultdict(dict)
 
@@ -87,34 +83,35 @@ def check_dns():
         t.start()
 
 
-def get_hover_results(domain, ipv):
+def get_resolver_results(domain, ipv):
     dns_config = ConfigResolver()
     dns_config.with_dict({
-        'provider_name': 'hover',
+        'provider_name': 'cloudflare',
         'action': 'list',
         'type': ipv,
         'domain': domain,
-        'hover': {
+        'cloudflare': {
-            'auth_username': HOVER_USERNAME,
+            'auth_username': CLOUDFLARE_USERNAME,
-            'auth_password': HOVER_PASSWORD,
+            'auth_token': CLOUDFLARE_AUTH_KEY,
         }
     })
     client = Client(dns_config)
     return client.execute()
 
 
-def set_hover_results(domain, ipv, new):
+def set_resolver_results(domain, ipv, new):
     dns_config = ConfigResolver()
     dns_config.with_dict({
-        'provider_name': 'hover',
+        'provider_name': 'cloudflare',
-        'action': 'update',
+        'action': 'create',
         'type': ipv,
         'domain': domain,
         'name': domain,
         'content': new,
-        'hover': {
+        'ttl': DEFAULT_TTL,
-            'auth_username': HOVER_USERNAME,
+        'cloudflare': {
-            'auth_password': HOVER_PASSWORD,
+            'auth_username': CLOUDFLARE_USERNAME,
+            'auth_token': CLOUDFLARE_AUTH_KEY,
         }
     })
     client = Client(dns_config)
@@ -122,7 +119,7 @@ def set_hover_results(domain, ipv, new):
 
 
 def get_dns(domain, ipv):
-    results = process_exp_backoff(get_hover_results, func_args=[domain, ipv])
+    results = process_exp_backoff(get_resolver_results, func_args=[domain, ipv])
     res = [x.get('content') for x in results if x.get('name') == domain]
     if len(res) == 1:
         print("Real {}: {}".format(ipv, res[0]))
@@ -130,7 +127,7 @@ def get_dns(domain, ipv):
 
 
 def set_dns(domain, ipv, new):
-    return process_exp_backoff(set_hover_results, func_args=[domain, ipv, new])
+    return process_exp_backoff(set_resolver_results, func_args=[domain, ipv, new])
 
 
 def get_uniq(_list):
@@ -139,6 +136,10 @@ def get_uniq(_list):
 
 
 def main():
+    with open('dns_addresses.csv') as csvf:
+        csvreader = csv.reader(csvf)
+        for row in csvreader:
+            dns_to_check.append(row)
     check_dns()
     check_observed()
     for t in tso:

provision/auto_dns/get_my_ip.py

@@ -1,6 +0,0 @@
-import sys
-
-
-if __name__ == "__main__":
-    domain = sys.argv[1]
-

provision/auto_dns/set_domain_ipv4.py

@@ -0,0 +1,40 @@
+import random
+import sys
+from time import sleep
+
+import requests
+
+from oshipka.util.os import run_os_cmd
+from oshipka.util.process import process_exp_backoff
+from provision.auto_dns.check import ipv4_sites, set_resolver_results, DEFAULT_TTL
+
+
+def verify_dns_domain(domain, ipv4):
+    ip_on_dns = run_os_cmd("dig +short {}".format(domain))
+    if not ip_on_dns:
+        raise Exception("Record for {} doesn't exist".format(domain))
+    ip_on_dns = ip_on_dns.strip()
+    assert ip_on_dns == ipv4
+
+
+def set_domain_ipv4(domain):
+    url = random.choice(ipv4_sites)
+    resp = requests.get(url)
+    ipv4 = resp.text.strip()
+    print('Got IP: {}'.format(ipv4))
+    try:
+        verify_dns_domain(domain, ipv4)
+        print("DNS already set up for {}".format(domain))
+    except Exception:
+        process_exp_backoff(set_resolver_results, func_args=[domain, 'A', ipv4])
+        print("Sleeping 5 sec...")
+        sleep(5)
+        print("Checking that {} is set".format(domain))
+        process_exp_backoff(verify_dns_domain, func_args=[domain, ipv4], max_attempts=0, max_sleep_time=DEFAULT_TTL)
+
+
+if __name__ == "__main__":
+    domain = sys.argv[1]
+    if not domain:
+        print('domain is required first argument')
+    set_domain_ipv4(domain)

provision/prod_mgmt.py

@@ -7,6 +7,7 @@ from provision.util import find_port, find_private_ipv4
 
 USER = "pi2"
 PARENT_DOMAIN = "pi2.dev"
+MAX_UPLOAD_SIZE = "10m"
 
 
 oshipka_path = os.environ.get('OSHIPKA_PATH')
@@ -33,23 +34,22 @@ def prod_install():
         project_name=project_name,
         project_domain=project_domain,
         upstream_ip=upstream_ip,
+        max_upload_size=MAX_UPLOAD_SIZE,
         port=port,
     )
 
-    service_tmpl = env.get_template('gunicorn.service')
+    tmpl_fname = [
-    service_txt = service_tmpl.render(ctx)
+        ('gunicorn.service', "{}.service".format(project_name)),
-    with open(os.path.join(TMP_PATH, "{}.service".format(project_name)), 'w') as f:
+        ('worker.service', "{}_worker.service".format(project_name)),
-        f.write(service_txt)
+        ('nginx_insecure.conf', "{}_insecure.conf".format(project_domain)),
+        ('nginx.conf', "{}.conf".format(project_domain)),
+    ]
 
-    worker_service_tmpl = env.get_template('worker.service')
+    for tmpl, fname in tmpl_fname:
-    worker_service_txt = worker_service_tmpl.render(ctx)
+        actual_tmpl = env.get_template(tmpl)
-    with open(os.path.join(TMP_PATH, "{}_worker.service".format(project_name)), 'w') as f:
+        rendered_tmpl = actual_tmpl.render(ctx)
-        f.write(worker_service_txt)
+        with open(os.path.join(TMP_PATH, fname), 'w') as f:
-
+            f.write(rendered_tmpl)
-    nginx_tmpl = env.get_template('nginx.conf')
-    nginx_txt = nginx_tmpl.render(ctx)
-    with open(os.path.join(TMP_PATH, "{}.conf".format(project_domain)), 'w') as f:
-        f.write(nginx_txt)
 
 
 if __name__ == "__main__":

provision/templates/nginx.conf

@@ -16,7 +16,7 @@ server {
 
     server_tokens off;
     charset utf-8;
-    client_max_body_size {{ project_domain }};
+    client_max_body_size {{ max_upload_size }};
 
     # certs sent to the client in SERVER HELLO are concatenated in ssl_certificate
     ssl_certificate /etc/letsencrypt/live/{{ project_domain }}/fullchain.pem;

provision/templates/nginx_insecure.conf

@@ -0,0 +1,15 @@
+server {
+    listen 80;
+    listen [::]:80;
+
+    server_name {{ project_domain }};
+
+    location / {
+        proxy_pass http://{{ upstream_ip }}:{{ port }};
+        proxy_redirect off;
+
+	    proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+}