Skip to content

Commit ef1f8a9

Browse files
mrekuccimrekucci
committed
fix: introduce backups for disaster recovery
1 parent 418c1af commit ef1f8a9

6 files changed

Lines changed: 160 additions & 2 deletions

File tree

‎infrastructure/nomad/playbooks/deploy.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -725,6 +725,10 @@
725725
break
726726
fi
727727
;;
728+
null)
729+
# This is a special case for periodic cron jobs.
730+
break
731+
;;
728732
*)
729733
{% if env != 'devenv' %}
730734
break

‎infrastructure/nomad/playbooks/init.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@
106106
fail_msg: "The environment name is not set correctly."
107107
success_msg: "The environment name is set to: {{ env }}."
108108

109-
- name: Ensure "{{ ansible_user_home }}/{{ env }}" Directory Exists
109+
- name: Ensure "{{ ansible_user_home }}/{{ env }}/artifacts" Directory Exists
110110
ansible.builtin.file:
111111
path: "{{ ansible_user_home }}/{{ env }}/artifacts"
112112
state: directory
@@ -115,6 +115,16 @@
115115
become: true
116116
become_user: "{{ ansible_user }}"
117117

118+
- name: Ensure "{{ ansible_user_home }}/{{ env }}/backups" Directory Exists
119+
ansible.builtin.file:
120+
path: "{{ ansible_user_home }}/{{ env }}/backups"
121+
state: directory
122+
mode: "0744"
123+
recurse: yes
124+
become: true
125+
become_user: "{{ ansible_user }}"
126+
when: env == "testenv" or env == "mainenv"
127+
118128
- name: Ensure "/var/lib/mev-commit/{{ env }}" Directory Exists
119129
ansible.builtin.file:
120130
path: "/var/lib/mev-commit/{{ env }}"

‎infrastructure/nomad/playbooks/templates/jobs/artifacts.nomad.j2‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ job "artifacts-{{ environments[env].version }}" {
99
# The cron corresponds to January 1st, 2100 and
1010
# it prevents the job from being run and cleaned up.
1111
periodic {
12-
cron = "0 0 1 1 6"
12+
crons = ["0 0 1 1 6"]
1313
}
1414
{% endif %}
1515

‎infrastructure/nomad/playbooks/templates/jobs/backup.nomad.j2‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
#jinja2: trim_blocks:True, lstrip_blocks:True
2+
job "backup" {
3+
datacenters = ["{{ datacenter }}"]
4+
5+
type = "batch"
6+
7+
# The cron will run the job every hour.
8+
periodic {
9+
crons = ["0 * * * *"]
10+
}
11+
12+
group "backup-group" {
13+
count = 1
14+
15+
network {
16+
mode = "bridge"
17+
18+
dns {
19+
servers = {{ (ansible_facts['dns']['nameservers'] + ['1.1.1.1']) | tojson }}
20+
}
21+
}
22+
23+
volume "data-volume" {
24+
type = "host"
25+
source = "data-volume"
26+
read_only = true
27+
}
28+
29+
volume "backups-volume" {
30+
type = "host"
31+
source = "backups-volume"
32+
read_only = false
33+
}
34+
35+
task "backup" {
36+
driver = "exec"
37+
38+
{% if env == 'devenv' %}
39+
resources {
40+
memory = 2048
41+
}
42+
{% endif %}
43+
44+
volume_mount {
45+
volume = "data-volume"
46+
destination = "/local/data"
47+
read_only = false
48+
}
49+
50+
volume_mount {
51+
volume = "backups-volume"
52+
destination = "/local/backups"
53+
read_only = false
54+
}
55+
56+
{% if env != 'devenv' %}
57+
artifact {
58+
source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit-geth_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz"
59+
}
60+
{% else %}
61+
artifact {
62+
source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/mev-commit-geth_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz"
63+
}
64+
{% endif %}
65+
66+
template {
67+
data = <<-EOH
68+
{%- raw %}
69+
GETH_DATA_DIR="/local/data/mev-commit-geth-member-node/node-{{ env "NOMAD_ALLOC_INDEX" }}"
70+
{% endraw %}
71+
GETH_LOG_FORMAT="{{ job.env.get('log-format', 'json') }}"
72+
GETH_LOG_TAGS="{{ 'service.name:' + job.name + '-{{ env "NOMAD_ALLOC_INDEX" }}' + ',service.version:' + version }}"
73+
EOH
74+
destination = "secrets/.env"
75+
env = true
76+
}
77+
78+
template {
79+
data = <<-EOH
80+
#!/usr/bin/env bash
81+
82+
{%- raw %}
83+
{{- range nomadService "datadog-agent-logs-collector" }}
84+
{{ if contains "tcp" .Tags }}
85+
exec > >(nc {{ .Address }} {{ .Port }}) 2>&1
86+
{{ end }}
87+
{{- end }}
88+
{% endraw %}
89+
90+
set -x #TODO: remove me!
91+
BACKUP_FILE="local/backups/{{ version }}_{{ job.name }}-{% raw %}{{ env "NOMAD_ALLOC_INDEX" }}{% endraw %}_$(date +%Y%m%d%H%M%S).rlp"
92+
if [[ ! -f /local/latest_block.txt ]]; then
93+
echo "0" > /local/latest_block.txt
94+
fi
95+
START_BLOCK=$(cat /local/latest_block.txt)
96+
END_BLOCK=$(local/mev-commit-geth attach --datadir="${GETH_DATA_DIR}" --exec "eth.blockNumber" 2>/dev/null)
97+
98+
echo "Exporting chain data to backup file: ${BACKUP_FILE}"
99+
START_TIME=$(date +%s)
100+
chmod +x local/mev-commit-geth
101+
local/mev-commit-geth \
102+
--verbosity=5 \
103+
--log.format="${GETH_LOG_FORMAT}" \
104+
--log.tags="${GETH_LOG_TAGS}" \
105+
--datadir="${GETH_DATA_DIR}" \
106+
export ${BACKUP_FILE} \
107+
${START_BLOCK} \
108+
${END_BLOCK}
109+
110+
if [[ "$?" -eq 0 ]] && [[ -f "${BACKUP_FILE}" ]]; then
111+
ELAPSED_TIME=$(($(date +%s) - START_TIME))
112+
echo "Backup finished in: $(date -u -d@${ELAPSED_TIME} +%H:%M:%S)"
113+
echo "Backup file size: $(du -h ${BACKUP_FILE} | cut -f1)"
114+
echo $((END_BLOCK + 1)) > /local/latest_block.txt
115+
else
116+
echo "Backup failed"
117+
rm -f ${BACKUP_FILE}
118+
exit 1
119+
fi
120+
EOH
121+
destination = "local/run.sh"
122+
change_mode = "noop"
123+
perms = "0755"
124+
}
125+
126+
config {
127+
command = "bash"
128+
args = ["-c", "exec local/run.sh"]
129+
}
130+
}
131+
}
132+
}

‎infrastructure/nomad/playbooks/templates/services/nomad.hcl.j2‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,11 @@ client {
3636
path = "{{ ansible_user_home }}/{{ env }}/artifacts"
3737
}
3838
{% endif %}
39+
{% if env == "testenv" or env == "mainenv" %}
40+
host_volume "backups-volume" {
41+
path = "{{ ansible_user_home }}/{{ env }}/backups"
42+
}
43+
{% endif %}
3944
}
4045
{% endif %}
4146

‎infrastructure/nomad/playbooks/variables/profiles.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -673,6 +673,12 @@ jobs:
673673
to: 8080
674674
env:
675675

676+
backup: &backup_job
677+
name: backup
678+
template: backup.nomad.j2
679+
env:
680+
log-format: "json"
681+
676682
profiles:
677683
ci:
678684
jobs:
@@ -735,6 +741,7 @@ profiles:
735741
- *mev_commit_oracle_job
736742
- *mev_commit_provider_emulator_node1_job
737743
- *datadog_agent_metrics_collector_job
744+
- *backup_job
738745

739746
stressnet:
740747
jobs:

0 commit comments

Comments
 (0)