Skip to content
This repository was archived by the owner on Nov 23, 2024. It is now read-only.

Prepare the bootstrap.sh script for a staging environment #44

Open
karolba wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Prepare the bootstrap.sh script for a staging environment #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c0d0220
Prepare the bootstrap.sh script for a staging environment
karolba Aug 19, 2022
0a7d89b
Retrigger CI with newer .yml
karolba Aug 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 11 additions & 13 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,16 @@
# Docker jest do użycia na produkcji - do developowania użyj vagranta!
# Bazowane na https://github.com/robertdebock/docker-debian-systemd

# Do uruchomienia (z powodu systemd) potrzebne jest włączone systemd na hoście, i podanie
# do docker run argumentów: --tty --privileged --volume /sys/fs/cgroup:/sys/fs/cgroup:ro
# Dockerfile jest do użycia na produkcji/staging - do developowania
# użyj vagranta!
#
# Do uruchomienia (z powodu wykorzystania systemd) kontener działa
# tylko na Podmanie, nie Dockerze

# TODO: czy --tty jest potrzebne? w README.md repo docker-debian-systemd jest podane,
# ale może być tylko jako przykład - jednak pamiętam że gdzieś w internecie pisali o tym
# że jest potrzebne przy systemd
# Budowanie wersji na produkcję: podman build .
# Budowanie wersji na staging: podman build --build-arg SWV2_ENV=staging .

FROM debian:11


# TODO: to coś daje/ma sens? zaimportowane z https://github.com/robertdebock/docker-debian-systemd
ENV container docker
# Bazowane na https://github.com/robertdebock/docker-debian-systemd
ENV DEBIAN_FRONTEND noninteractive

RUN bash -c 'shopt -s nullglob && \
apt-get update && \
apt-get install -y systemd systemd-sysv && \
Expand All @@ -28,7 +24,9 @@ RUN bash -c 'shopt -s nullglob && \
/lib/systemd/system/systemd-update-utmp*'

ADD . /opt/sw
RUN /opt/sw/bootstrap.sh --prod

ARG SWV2_ENV=production
RUN /opt/sw/bootstrap.sh --$SWV2_ENV

VOLUME [ "/sys/fs/cgroup" ]
CMD ["/lib/systemd/systemd"]
2 changes: 1 addition & 1 deletion Vagrantfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ Vagrant.configure("2") do |config|

# Use a script to provision the VM
config.vagrant.plugins = [ "vagrant-vbguest" ]
config.vm.provision :shell, path: "bootstrap.sh", run: "always"
config.vm.provision :shell, path: "bootstrap.sh", args: ["--development"], run: "always"

# Only reset the database when provisioning explicitly
config.vm.provision :shell, path: "sw-database/resetdb.sh"
Expand Down
262 changes: 163 additions & 99 deletions bootstrap.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,142 +12,206 @@
set -xeu
shopt -s nullglob

prod=0
if [[ $# -ge 1 && $1 == --prod ]]; then
prod=1
fi

# Na dockerze/podmanie przy konfiguracji systemd nie jest dostępne - nie działa `systemctl enable`
# można to obejść manualnie tworząc symlink w /etc/systemd/system/{target}.wants/{unit} do pliku z unitem
systemctl_enable() {
local unit_name

if [[ -d /run/systemd/system ]]; then
systemctl enable "$@"
if [[ $1 == --now ]]; then
shift

for unit_name; do
if systemctl cat "$unit_name" | grep -q '^\[Install\]'; then
if systemctl is-failed "$unit_name" >/dev/null; then
# Przy re-provisioningu vagranta resetujemy status failed usługi który powoduje
# że przy szybkim restartowaniu systemd nie włącza usługi spowrotem
systemctl reset-failed "$unit_name"
fi
systemctl restart "$unit_name" || true
systemctl enable --now "$unit_name" || true
fi
done
else
systemctl enable "$@"
fi
else
if [[ $1 == --now ]]; then
shift
fi

local unit_name="$1"
local unit_file="/etc/systemd/system/$unit_name"
local wanted_by=$(< "$unit_file" sed 's/\s*#.*//' | awk -F'=' '/^\[.*\]$/{section=$0;next} section=="[Install]" { $1=""; print }')
local target
for target in $wanted_by; do
local target_path="/etc/systemd/system/${target}.wants/"
local symlink_path="$target_path/$unit_name"
local symlink_target="/etc/systemd/system/$unit_name"
mkdir -p "$target_path"
ln -s "$symlink_target" "$symlink_path"
# Na dockerze/podmanie przy konfiguracji systemd nie jest dostępne - nie działa `systemctl enable`
# można to obejść manualnie tworząc symlink w /etc/systemd/system/{target}.wants/{unit} do pliku z unitem
for unit_name; do
local unit_file="/etc/systemd/system/$unit_name"
local wanted_by=$(< "$unit_file" sed 's/\s*#.*//' | awk -F'=' '/^\[.*\]$/{section=$0;next} section=="[Install]" { $1=""; print }')
local target
for target in $wanted_by; do
local target_path="/etc/systemd/system/${target}.wants/"
local symlink_path="$target_path/$unit_name"
local symlink_target="/etc/systemd/system/$unit_name"
mkdir -p "$target_path"
ln -s "$symlink_target" "$symlink_path"
done
done
fi
}

install-general-packages() {
# Do an apt-get update only if there wasn't one in the last 3 hours (makes vagrant provision faster)
if [ -z "$(find /var/cache/apt -maxdepth 0 -mmin -180)" ]; then
apt-get update
fi

apt-get install -y --no-install-recommends \
git wget gnupg ca-certificates less procps sudo htop gunicorn gawk \
python3 python3-flask python3-flask-login python3-psycopg2 python3-bcrypt python3-pip python3-tz python3-dateutil
pip3 install furl
for requirements_file in /opt/sw/*/requirements.txt; do
pip3 install -r "$requirements_file"
done
}

if ! ((prod)); then
setup-vagrant-user() {
# Allow login to root like to the vagrant user
mkdir -p /root/.ssh
cp /home/vagrant/.ssh/authorized_keys /root/.ssh/
chown root:root /root/.ssh /root/.ssh/authorized_keys
chmod 600 /root/.ssh /root/.ssh/authorized_keys
fi
}

# Do an apt-get update only if there wasn't one in the last 3 hours (makes vagrant provision faster)
if [ -z "$(find /var/cache/apt -maxdepth 0 -mmin -180)" ]; then
apt-get update
fi
vagrant-windows-ntfs-workaround() {
# Mount runtime directories as tmpfs so this works when ran on windows
for dir in /opt/sw/{poll,v,v-archive,logs}; do
mount -t tmpfs tmpfs "$dir" &
done
}

apt-get install -y --no-install-recommends \
git wget gnupg ca-certificates less procps sudo mailutils htop gunicorn gawk \
python3 python3-flask python3-flask-login python3-psycopg2 python3-bcrypt python3-pip python3-tz python3-dateutil
setup-postgresql() {
apt-get install -y --no-install-recommends postgresql postgresql-client
systemctl_enable --now postgresql.service || true
}

if ! ((prod)); then
apt-get install -y --no-install-recommends postgresql postgresql-client
fi
setup-openresty() {
# Install openresty if not already installed
if ! dpkg -l openresty &> /dev/null; then
wget -O - https://openresty.org/package/pubkey.gpg | apt-key add - \
&& codename=`grep -Po 'VERSION="[0-9]+ \(\K[^)]+' /etc/os-release` \
&& echo "deb http://openresty.org/package/debian $codename openresty" | tee /etc/apt/sources.list.d/openresty.list \
&& apt-get update \
&& apt-get -y install --no-install-recommends openresty
fi

# Install openresty if not already installed
if ! dpkg -l openresty &> /dev/null; then
wget -O - https://openresty.org/package/pubkey.gpg | apt-key add - \
&& codename=`grep -Po 'VERSION="[0-9]+ \(\K[^)]+' /etc/os-release` \
&& echo "deb http://openresty.org/package/debian $codename openresty" | tee /etc/apt/sources.list.d/openresty.list \
&& apt-get update \
&& apt-get -y install --no-install-recommends openresty
fi
unlink /etc/openresty/nginx.conf
ln -s /opt/sw/sw-openresty/nginx.conf /etc/openresty/nginx.conf

unlink /etc/openresty/nginx.conf
ln -s /opt/sw/sw-openresty/nginx.conf /etc/openresty/nginx.conf
apt-get install -y --no-install-recommends openresty-opm
opm get 3scale/lua-resty-url

apt-get install -y openresty-opm
opm get 3scale/lua-resty-url
pip3 install furl
systemctl_enable --now openresty.service || true
}

if ! ((prod)); then
setup-postfix() {
apt-get install -y --no-install-recommends mailutils postfix
rm -f /etc/postfix/{main.cf,password}
ln -s /opt/sw/sw-postfix/main.cf /etc/postfix/

if ! [[ -f /etc/mailname ]]; then
echo wybory.samorzad.pwr.edu.pl > /etc/mailname
fi
cp /opt/sw/sw-postfix/password /etc/postfix/
postmap /etc/postfix/password # Can't do it on a symlink

systemctl enable --now postgresql || true
fi
systemctl_enable --now postfix.service || true
}

for requirements_file in /opt/sw/*/requirements.txt; do
pip3 install -r "$requirements_file"
done
create-runtime-dirs() {
mkdir -p /opt/sw/{v,v-archive,poll,logs}
chown -R www-data:www-data /opt/sw/{v,poll,logs}
}

mkdir -p /opt/sw/{v,v-archive,poll,logs}
chown -R www-data:www-data /opt/sw/{v,poll,logs}
copy-enable-unit-files-make-utility-scripts() {
local services_unit_files=( /opt/sw/*/*.{service,timer,socket} )
cp -t /etc/systemd/system/ -- "${services_unit_files[@]}"
if [[ -d /run/systemd/system ]]; then
systemctl daemon-reload
fi

# Start up all systemd services
local sw_services=()
local service_unit_file
for service_unit_file in "${services_unit_files[@]}"; do
sw_services+=( "$(basename "$service_unit_file")" )
done
systemctl_enable --now "${sw_services[@]}"

make-script() {
echo "#!/bin/sh
$2" > "/usr/local/bin/$1"
chmod +x "/usr/local/bin/$1"
}

local extra_services=()
if ((enable_postgresql)); then extra_services+=(postgresql); fi
if ((enable_postfix)); then extra_services+=(postfix); fi
if ((enable_openresty)); then extra_services+=(openresty); fi

make-script sw-logs "journalctl -e -b $(printf -- '-u %q"*" ' "${extra_services[@]}") -u 'sw-*' --lines=all --follow"
make-script sw-status "systemctl status -l --no-pager --lines=100 $(printf '%q ' "${sw_services[@]}" "${extra_services[@]}")"
make-script sw-restart "systemctl reset-failed $(printf '%q ' "${sw_services[@]}"); systemctl restart $(printf '%q ' "${sw_services[@]}" "${extra_services[@]}")"
}

services_unit_files=( /opt/sw/*/*.{service,timer,socket} )

cp -t /etc/systemd/system/ -- "${services_unit_files[@]}"
if ! ((prod)); then
systemctl daemon-reload
case "${1: }" in
--production)
# Runs when setting up a Podman container from the master branch to prepare for production
enable_postgresql=0
enable_postfix=0
enable_openresty=0
vagrant_quirks=0
;;
--staging)
# Runs when setting up a Podman container from a pull request branch for the live environment
enable_postgresql=1
enable_postfix=1
enable_openresty=1
vagrant_quirks=0
;;
--development)
# Runs when setting up a development vagrant machine
enable_postgresql=1
enable_postfix=1
enable_openresty=1
vagrant_quirks=1
;;
*)
set +x
{
echo "Usage: $0 [--production|--staging|--development]"
echo ""
echo " --production - setup a Podman container for production deployment"
echo " (no mail server, database and http server)"
echo ""
echo " --staging - setup a Podman container for staging - automatic Pull"
echo " Request deployments (include mail server, database and webserver)"
echo ""
echo " --development - setup a local Vagrant box"
} >> /dev/stderr
exit 1
;;
esac

install-general-packages
if ((vagrant_quirks)); then
setup-vagrant-user
vagrant-windows-ntfs-workaround
fi

extra_services=()
if ! ((prod)); then
extra_services+=( openresty postfix )
if ((enable_postgresql)); then
setup-postgresql
fi

all_services=()
for service_unit_file in "${services_unit_files[@]}" "${extra_services[@]}"; do
all_services+=( "$(basename "$service_unit_file")" )
done

if (( prod )); then
for service_name in "${all_services[@]}"; do
systemctl_enable "$service_name"
done
else
for service_name in "${all_services[@]}"; do
{
if systemctl cat "$service_name" | grep -q '^\[Install\]'; then
if systemctl is-failed "$service_name" >/dev/null; then
systemctl reset-failed "$service_name"
fi
systemctl restart "$service_name" || true
systemctl enable --now "$service_name" || true
fi
} &
done
if ((enable_openresty)); then
setup-openresty
fi

make-script() {
echo "#!/bin/sh
$2" > "/usr/local/bin/$1"
chmod +x "/usr/local/bin/$1"
}

make-script sw-logs "journalctl -e -b $(printf -- '-u %q"*" ' "${extra_services[@]}") -u 'sw-*' --lines=all --follow"
make-script sw-status "systemctl status -l --no-pager --lines=100 $(printf '%q ' "${all_services[@]}")"
make-script sw-restart "systemctl reset-failed $(printf '%q ' "${all_services[@]}"); systemctl restart $(printf '%q ' "${all_services[@]}")"

if ! ((prod)); then
# Mount runtime directories as tmpfs so this works when ran on windows
for dir in /opt/sw/{poll,v,v-archive,logs}; do
mount -t tmpfs tmpfs "$dir" &
done
wait # wait for this "&" and previous one - when enabling systemd services
if ((enable_postfix)); then
setup-postfix
fi
copy-enable-unit-files-make-utility-scripts

wait