daniel
/
oshipka
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

testing setting up a site end-2-end

This commit is contained in:
Daniel Tsvetkov 2020-06-21 17:45:33 +02:00
parent 03c2addf8a
commit ed841fd033
8 changed files with 128 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -5,4 +5,4 @@ data/db.sqlite
__pycache__
oshipka.egg-info
provision/tmp
secrets.py
provision/auto_dns/sensitive.py

53
oshipka.sh
View File

@ -64,13 +64,9 @@ init() {
}
install_cert() {
#[ ! -f /tmp/certbot-auto ] && wget -O /tmp/certbot-auto https://dl.eff.org/certbot-auto
#chmod +x /tmp/certbot-auto
#certbot=/tmp/certbot-auto
shift
PROJECT_DOMAIN=$1
sudo apt install certbot
certbot certonly --authenticator standalone --installer nginx --pre-hook "service nginx stop" --post-hook "service nginx start" --redirect --agree-tos --no-eff-email --email danieltcv@gmail.com -d ${PROJECT_DOMAIN} --no-bootstrap
sudo certbot certonly --dry-run --authenticator standalone --installer nginx --pre-hook "service nginx stop" --post-hook "service nginx start" --redirect --agree-tos --no-eff-email --email danieltcv@gmail.com -d ${PROJECT_DOMAIN} --no-bootstrap
}
bootstrap() {
@ -123,11 +119,12 @@ run_in_prod() {
}
prod_install() {
set -e
shift
source venv/bin/activate
PROJECT_NAME=$(basename `pwd`)
echo "1/4 Generating service and config files..."
"${OSHIPKA_PATH}/venv/bin/python" "${OSHIPKA_PATH}/provision/prod_mgmt.py"
echo "1/6 Generating service and config files..."
python "${OSHIPKA_PATH}/provision/prod_mgmt.py"
if [ -f "/etc/systemd/system/${PROJECT_NAME}.service" ]; then
echo "Service gunicorn for ${PROJECT_NAME} service exists."
systemctl status ${PROJECT_NAME}
@ -138,7 +135,7 @@ prod_install() {
sudo systemctl start "${PROJECT_NAME}"
fi
echo "2/5 Installing '$PROJECT_NAME' worker service"
echo "2/6 Installing '$PROJECT_NAME' worker service"
if [ -f "/etc/systemd/system/${PROJECT_NAME}_worker.service" ]; then
echo "Service worker for ${PROJECT_NAME} service exists."
systemctl status "${PROJECT_NAME}_worker"
@ -148,17 +145,38 @@ prod_install() {
sudo systemctl start "${PROJECT_NAME}_worker"
fi
# TODO: Update DNS
NGINX_CONFIG_FILE=$(basename `find $OSHIPKA_PATH/provision/tmp -name *.conf`)
DOMAIN=$(basename -s .conf $NGINX_CONFIG_FILE)
echo "3/5 Installing '$DOMAIN' domain..."
"${OSHIPKA_PATH}/venv/bin/python" "${OSHIPKA_PATH}/audo_dns/set_domain.py" "$DOMAIN"
echo "3/6 Installing '$DOMAIN' domain..."
python "${OSHIPKA_PATH}/provision/auto_dns/set_domain_ipv4.py" "$DOMAIN"
# TODO: Create certificate
echo "4/5 Installing '$PROJECT_NAME' certificate..."
sudo apt install nginx
sudo systemctl start nginx
echo "Enabling firewall rule for 192.168.1.1 -> 80/tcp..."
sudo ufw allow proto tcp from 192.168.1.1 to any port 80
echo "4/6 Installing '$PROJECT_NAME' insecure nginx config..."
if [ -f "/etc/nginx/sites-available/${DOMAIN}_insecure.conf" ]; then
echo "Insecure Nginx config for ${PROJECT_NAME} available."
if [ -f "/etc/nginx/sites-enabled/${DOMAIN}_insecure.conf" ]; then
echo "Nginx config for ${PROJECT_NAME} enabled."
else
echo "Nginx config for ${PROJECT_NAME} NOT enabled."
fi
else
echo "Installing insecure nginx config for ${PROJECT_NAME} -> enabling + available."
sudo cp "${OSHIPKA_PATH}/provision/tmp/${DOMAIN}_insecure.conf" /etc/nginx/sites-available/
sudo ln -s "/etc/nginx/sites-available/${DOMAIN}_insecure.conf" "/etc/nginx/sites-enabled/${DOMAIN}_insecure.conf"
sudo systemctl reload nginx
fi
echo "5/6 Installing '$PROJECT_NAME' certificate..."
install_cert $DOMAIN
echo "5/5 Installing '$PROJECT_NAME' nginx config..."
echo "6/6 Installing '$PROJECT_NAME' secure nginx config..."
echo "Enabling firewall rule for 192.168.1.1 -> 443/tcp..."
sudo ufw allow proto tcp from 192.168.1.1 to any port 443
echo "Removing '$PROJECT_NAME' insecure nginx config..."
sudo rm "/etc/nginx/sites-available/${DOMAIN}_insecure.conf" "/etc/nginx/sites-enabled/${DOMAIN}_insecure.conf"
if [ -f "/etc/nginx/sites-available/${NGINX_CONFIG_FILE}" ]; then
echo "Nginx config for ${PROJECT_NAME} available."
if [ -f "/etc/nginx/sites-enabled/${NGINX_CONFIG_FILE}" ]; then
@ -166,10 +184,11 @@ prod_install() {
else
echo "Nginx config for ${PROJECT_NAME} NOT enabled."
fi
else
else
echo "Installing nginx config for ${PROJECT_NAME} -> enabling + available."
sudo cp "${OSHIPKA_PATH}/provision/tmp/${NGINX_CONFIG_FILE}" /etc/nginx/sites-enabled/
sudo cp "${OSHIPKA_PATH}/provision/tmp/${NGINX_CONFIG_FILE}" /etc/nginx/sites-available/
sudo ln -s "/etc/nginx/sites-available/${NGINX_CONFIG_FILE}" "/etc/nginx/sites-enabled/${NGINX_CONFIG_FILE}"
sudo systemctl reload nginx
fi
}
@ -256,7 +275,7 @@ command_main() {
;;
prod_install) prod_install "$@"
;;
cert) install_cert "$@"
cert) shift && install_cert "$@"
;;
*) >&2 echo -e "${HELP}"
return 1

43
provision/auto_dns/check.py
View File

@ -11,10 +11,12 @@ from lexicon.client import Client
from lexicon.config import ConfigResolver
from oshipka.util.process import process_exp_backoff
from secrets import HOVER_USERNAME, HOVER_PASSWORD
from sensitive import CLOUDFLARE_USERNAME, CLOUDFLARE_AUTH_KEY
SLEEP_SECONDS = 60 * 30 # 30 min
DEFAULT_TTL = 5 * 60
ipv4_sites = [
"http://www.icanhazip.com",
"http://ipecho.net/plain",
@ -24,19 +26,13 @@ ipv4_sites = [
]
ipv6_sites = [
"http://bot.whatismyipaddress.com",
"https://diagnostic.opendns.com/myip",
"https://ifconfig.co/ip",
]
dns_to_check = []
with open('dns_addresses.csv') as csvf:
csvreader = csv.reader(csvf)
for row in csvreader:
dns_to_check.append(row)
ip_checkers = ipv4_sites + ipv6_sites
dns_to_check = []
results = []
real = defaultdict(dict)
@ -87,34 +83,35 @@ def check_dns():
t.start()
def get_hover_results(domain, ipv):
def get_resolver_results(domain, ipv):
dns_config = ConfigResolver()
dns_config.with_dict({
'provider_name': 'hover',
'provider_name': 'cloudflare',
'action': 'list',
'type': ipv,
'domain': domain,
'hover': {
'auth_username': HOVER_USERNAME,
'auth_password': HOVER_PASSWORD,
'cloudflare': {
'auth_username': CLOUDFLARE_USERNAME,
'auth_token': CLOUDFLARE_AUTH_KEY,
}
})
client = Client(dns_config)
return client.execute()
def set_hover_results(domain, ipv, new):
def set_resolver_results(domain, ipv, new):
dns_config = ConfigResolver()
dns_config.with_dict({
'provider_name': 'hover',
'action': 'update',
'provider_name': 'cloudflare',
'action': 'create',
'type': ipv,
'domain': domain,
'name': domain,
'content': new,
'hover': {
'auth_username': HOVER_USERNAME,
'auth_password': HOVER_PASSWORD,
'ttl': DEFAULT_TTL,
'cloudflare': {
'auth_username': CLOUDFLARE_USERNAME,
'auth_token': CLOUDFLARE_AUTH_KEY,
}
})
client = Client(dns_config)
@ -122,7 +119,7 @@ def set_hover_results(domain, ipv, new):
def get_dns(domain, ipv):
results = process_exp_backoff(get_hover_results, func_args=[domain, ipv])
results = process_exp_backoff(get_resolver_results, func_args=[domain, ipv])
res = [x.get('content') for x in results if x.get('name') == domain]
if len(res) == 1:
print("Real {}: {}".format(ipv, res[0]))
@ -130,7 +127,7 @@ def get_dns(domain, ipv):
def set_dns(domain, ipv, new):
return process_exp_backoff(set_hover_results, func_args=[domain, ipv, new])
return process_exp_backoff(set_resolver_results, func_args=[domain, ipv, new])
def get_uniq(_list):
@ -139,6 +136,10 @@ def get_uniq(_list):
def main():
with open('dns_addresses.csv') as csvf:
csvreader = csv.reader(csvf)
for row in csvreader:
dns_to_check.append(row)
check_dns()
check_observed()
for t in tso:

6
provision/auto_dns/get_my_ip.py
View File

@ -1,6 +0,0 @@
import sys
if __name__ == "__main__":
domain = sys.argv[1]

40
provision/auto_dns/set_domain_ipv4.py Normal file
View File

@ -0,0 +1,40 @@
import random
import sys
from time import sleep
import requests
from oshipka.util.os import run_os_cmd
from oshipka.util.process import process_exp_backoff
from provision.auto_dns.check import ipv4_sites, set_resolver_results, DEFAULT_TTL
def verify_dns_domain(domain, ipv4):
ip_on_dns = run_os_cmd("dig +short {}".format(domain))
if not ip_on_dns:
raise Exception("Record for {} doesn't exist".format(domain))
ip_on_dns = ip_on_dns.strip()
assert ip_on_dns == ipv4
def set_domain_ipv4(domain):
url = random.choice(ipv4_sites)
resp = requests.get(url)
ipv4 = resp.text.strip()
print('Got IP: {}'.format(ipv4))
try:
verify_dns_domain(domain, ipv4)
print("DNS already set up for {}".format(domain))
except Exception:
process_exp_backoff(set_resolver_results, func_args=[domain, 'A', ipv4])
print("Sleeping 5 sec...")
sleep(5)
print("Checking that {} is set".format(domain))
process_exp_backoff(verify_dns_domain, func_args=[domain, ipv4], max_attempts=0, max_sleep_time=DEFAULT_TTL)
if __name__ == "__main__":
domain = sys.argv[1]
if not domain:
print('domain is required first argument')
set_domain_ipv4(domain)

26
provision/prod_mgmt.py
View File

@ -7,6 +7,7 @@ from provision.util import find_port, find_private_ipv4
USER = "pi2"
PARENT_DOMAIN = "pi2.dev"
MAX_UPLOAD_SIZE = "10m"
oshipka_path = os.environ.get('OSHIPKA_PATH')
@ -33,23 +34,22 @@ def prod_install():
project_name=project_name,
project_domain=project_domain,
upstream_ip=upstream_ip,
max_upload_size=MAX_UPLOAD_SIZE,
port=port,
)
service_tmpl = env.get_template('gunicorn.service')
service_txt = service_tmpl.render(ctx)
with open(os.path.join(TMP_PATH, "{}.service".format(project_name)), 'w') as f:
f.write(service_txt)
tmpl_fname = [
('gunicorn.service', "{}.service".format(project_name)),
('worker.service', "{}_worker.service".format(project_name)),
('nginx_insecure.conf', "{}_insecure.conf".format(project_domain)),
('nginx.conf', "{}.conf".format(project_domain)),
]
worker_service_tmpl = env.get_template('worker.service')
worker_service_txt = worker_service_tmpl.render(ctx)
with open(os.path.join(TMP_PATH, "{}_worker.service".format(project_name)), 'w') as f:
f.write(worker_service_txt)
nginx_tmpl = env.get_template('nginx.conf')
nginx_txt = nginx_tmpl.render(ctx)
with open(os.path.join(TMP_PATH, "{}.conf".format(project_domain)), 'w') as f:
f.write(nginx_txt)
for tmpl, fname in tmpl_fname:
actual_tmpl = env.get_template(tmpl)
rendered_tmpl = actual_tmpl.render(ctx)
with open(os.path.join(TMP_PATH, fname), 'w') as f:
f.write(rendered_tmpl)
if __name__ == "__main__":

2
provision/templates/nginx.conf
View File

@ -16,7 +16,7 @@ server {
server_tokens off;
charset utf-8;
client_max_body_size {{ project_domain }};
client_max_body_size {{ max_upload_size }};
# certs sent to the client in SERVER HELLO are concatenated in ssl_certificate
ssl_certificate /etc/letsencrypt/live/{{ project_domain }}/fullchain.pem;

15
provision/templates/nginx_insecure.conf Normal file
View File

@ -0,0 +1,15 @@
server {
listen 80;
listen [::]:80;
server_name {{ project_domain }};
location / {
proxy_pass http://{{ upstream_ip }}:{{ port }};
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}