Setting up nginx+letsencrypt as a reverse proxy

This is always made slightly tricky by the fact that nginx requires the ssl certificates to be present in order to start up. This poses a chicken-and-egg problem, in that we require nginx to be serving the letsencrypt ACME challenge in order to retrieve the certificates in order to start nginx. We solve this simply by running the https server separately.

First, create the nginx configurations:

# nginx-ingress-http.conf
events {
}

http {
  include mime.types;

  server {
    listen 80;
    listen [::]:80;

    server_name sg.horlick.me;

    location /.well-known/acme-challenge/ {
      root /var/www/certbot;
    }

    location / {
      return 301 https://$host$request_uri;
    }    
  }
}

# nginx-ingress-https.conf
events {
}

http {
  include mime.types;

  server {
    listen 443 ssl;
    listen [::]:443 ssl;

    server_name sg.horlick.me;

    ssl_certificate /etc/letsencrypt/live/sg.horlick.me/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/sg.horlick.me/privkey.pem;

    # taken from https://github.com/certbot/certbot/blob/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf
    ssl_session_cache shared:le_nginx_SSL:10m;
    ssl_session_timeout 1440m;
    ssl_session_tickets off;

    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_prefer_server_ciphers off;

    ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384";

    ssl_dhparam /etc/ssl/certs/dhparam.pem;

    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;

    location / {
      proxy_pass http://backend:8080/;
      proxy_http_version  1.1;
      proxy_cache_bypass  $http_upgrade;

      proxy_set_header Host              $host;
      proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
      proxy_set_header X-Real-IP         $remote_addr;
      proxy_set_header X-Forwarded-Host  $host;
      proxy_set_header X-Forwarded-Proto $scheme;
      proxy_set_header X-Forwarded-Port  $server_port;
    }
  }
}

mkdir -p /etc/nginx
cp nginx-ingress-http.conf /etc/nginx/nginx-ingress-http.conf
cp nginx-ingress-https.conf /etc/nginx/nginx-ingress-https.conf

Create the http server:

# nginx-ingress-http.service
[Unit]
Description=http server that serves letsencrypt ACME challenges and 301 redirects to https 
After=docker.service
Requires=docker.service

[Service]
TimeoutStartSec=0
Restart=always
ExecStartPre=-/usr/bin/docker stop nginx-ingress-http
ExecStartPre=-/usr/bin/docker rm nginx-ingress-http
ExecStartPre=/usr/bin/docker pull nginx
ExecStart=/usr/bin/docker run -i --rm \
  --name nginx-ingress-http \
  --publish=80:80 \
  --volume /etc/letsencrypt:/etc/letsencrypt \
  --volume /var/www/certbot:/var/www/certbot \
  --volume /etc/nginx/nginx-ingress-http.conf:/etc/nginx/nginx.conf:ro \
  nginx

[Install]
WantedBy=multi-user.target

cp nginx-ingress-http.service /etc/systemd/system/nginx-ingress-http.service
systemctl enable nginx-ingress-http.service
systemctl start nginx-ingress-http.service
journalctl -u nginx-ingress-http.service

Now we're listening for ACME challenges, we can run certbot to generate the certificate.

# letsencrypt-generate-cert.service
[Unit]
Description=letsencrypt cert generation oneshot
Requires=docker.service

[Service]
Type=oneshot
ExecStart=/usr/bin/docker run -i --rm \
  --volume=/etc/letsencrypt:/etc/letsencrypt \
  --volume=/var/www/certbot:/var/www/certbot \
  certbot/certbot \
  certonly \
  --webroot \
  --register-unsafely-without-email \
  --agree-tos \
  --webroot-path=/var/www/certbot \
  -d sg.horlick.me

cp letsencrypt-generate-cert.service /etc/systemd/system/letsencrypt-generate-cert.service
systemctl start letsencrypt-generate-cert.service
journalctl -u letsencrypt-generate-cert.service

Create a timer that will auto-renew the certificate before it expires.

# letsencrypt-renew-cert.service
[Unit]
Description=letsencrypt cert update oneshot
Requires=docker.service

[Service]
Type=oneshot
ExecStart=/usr/bin/docker run -i --rm \
  --volume=/etc/letsencrypt:/etc/letsencrypt \
  --volume=/var/www/certbot:/var/www/certbot \
  certbot/certbot \
  renew
ExecStartPost=/usr/bin/docker exec \
  nginx-ingress-https \
  /bin/bash -c "nginx -s reload"

# letsencrypt-renew-cert.timer
[Unit]
Description=letsencrypt oneshot timer
Requires=docker.service

[Timer]
OnCalendar=daily

[Install]
WantedBy=timers.target

cp letsencrypt-renew-cert.service letsencrypt-renew-cert.timer /etc/systemd/system/
systemctl enable letsencrypt-renew-cert.timer
systemctl list-timers --all

Finally we can install and start the https server.

# nginx-ingress-https.service
[Unit]
Description=reverse proxy for api and static asset server
After=docker.service
Requires=docker.service

[Service]
TimeoutStartSec=0
Restart=always
ExecStartPre=-/usr/bin/docker stop nginx-ingress-https
ExecStartPre=-/usr/bin/docker rm nginx-ingress-https
ExecStartPre=/usr/bin/docker pull nginx
ExecStart=/usr/bin/docker run -i --rm \
  --name nginx-ingress-https \
  --publish=443:443 \
  --link=backend \
  --add-host=host.docker.internal:host-gateway \
  --volume /etc/nginx/nginx-ingress-https.conf:/etc/nginx/nginx.conf:ro \
  --volume /etc/letsencrypt:/etc/letsencrypt:ro \
  --volume /etc/ssl/certs/dhparam.pem:/etc/ssl/certs/dhparam.pem:ro \
  nginx

[Install]
WantedBy=multi-user.target

# create the Diffie–Hellman parameters
openssl dhparam -out /etc/ssl/certs/dhparam.pem 2048

# create the https server
cp nginx-ingress-https.service /etc/systemd/system/nginx-ingress-https.service
systemctl enable nginx-ingress-https.service
systemctl start nginx-ingress-https.service
journalctl -u nginx-ingress-https.service