diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/Makefile b/Makefile new file mode 100755 index 0000000..6ff235e --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +run: + npm start diff --git a/docs/10-deployment/20-helm-chart.md b/docs/10-deployment/20-helm-chart.md index c562f95..3adb4de 100755 --- a/docs/10-deployment/20-helm-chart.md +++ b/docs/10-deployment/20-helm-chart.md @@ -5,7 +5,7 @@ We provide a [Helm chart for backmon](https://github.com/dreitier/helm-charts/). You can either use that repository and install the helm chart with ```bash -$ git clone https://github.com/dreitier/helm-charts/` +$ git clone https://github.com/dreitier/helm-charts/ $ cd helm-charts/charts/backmon $ helm install backmon ``` diff --git a/docs/20-reference/10-backmon-configuration/01-overview.md b/docs/20-reference/10-backmon-configuration/01-overview.md index ce312bf..e7fd88c 100755 --- a/docs/20-reference/10-backmon-configuration/01-overview.md +++ b/docs/20-reference/10-backmon-configuration/01-overview.md @@ -55,9 +55,10 @@ environments: - "/regular_ex.*ssions_are_supported/" all_others: exclude env_2: - access_key_id: my_access_key_id - secret_access_key: my_secret_access_key - auto_discover_disks: true + s3: + access_key_id: my_access_key_id + secret_access_key: my_secret_access_key + auto_discover_disks: true disks: include: - my-bucket-1 @@ -70,31 +71,33 @@ environments: ## Available configuration keys -| Key | Default | Required | Description | -|------------------------------------------|-------------------------------------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `port` | `80` (*int*) | No | Default HTTP port to listen for requests. TLS is not supported at the moment. Consider using a proxy if you need encryption. | -| `update_interval` | `1h` (*duration*) | No | Checks each disk in that duration interval. [time.ParseDuration format](https://pkg.go.dev/time#example-ParseDuration) must be used. | -| `log_level` | `` (*one of `debug`, `info`*) | No | Used log level; will be overwritten if `--debug` is used. | -| `downloads.enabled` | `false` | No | If `true`, the latest artifact of a monitored backup disk can be downloaded. This is disabled by default for security reasons ([#1](https://github.com/dreitier/backmon/issues/1)). | -| `http.basic_auth.username` | `` (*string*) | No | Username for HTTP Basic Authentication. If this is set, `http.basic_auth.password` must be also set. | -| `http.basic_auth.password` | `` (*string*) | No | Password for HTTP Basic Authentication. If this is set, `http.basic_auth.username` must be also set. | -| `http.tls.certificate` | `` (*string*) | No | Path to certificate file. If this is set, `http.tls.key` must be also set. | -| `http.tls.key` | `` (*string*) | No | Path to private key file. If this is set, `http.tls.certificate` must be also set. | -| `http.tls.strict` | `false` (*bool*) | No | If set to true, a preferred TLS default configuration is used. | -| `environments` | `` (*list of environment*) | No | Each `environment` to check. | -| `environments[]` | `` (*string*) | __Yes__ | Name of environment. | -| `environments[$env].definitions` | `backup_definitions.yaml` (*string*) | No | YAML file containing the backup definitions. | -| `environments[$env].path` | `` (*string*) | No | Local path to check for. If you use the `path` parameter, other parameters specific for S3 are ignored. | -| `environments[$env].region` | `eu-central-1` (*string*) | No | AWS region | -| `environments[$env].force_path_style` | `false` (*bool*) | No | Use path-style for that S3 bucket. This is deprecated by AWS S3 and should be probably `false`. | -| `environments[$env].access_key_id` | `` (*string*) | __Yes__ | AWS Access Key | -| `environments[$env].secret_access_key` | `` (*string*) | __Yes__ | AWS Secret Access Key | -| `environments[$env].endpoint` | `` (*string*) | No | Custom AWS S3 endpoint. This must be used for Minio buckets or if you are using a local S3 instance. | -| `environments[$env].token` | `` (*string*) | No | AWS STS session token. You can leave that empty. | -| `environments[$env].auto_discover_disks` | `true` (*bool*) | No | Automatically iterate over each S3 bucket. | -| `environments[$env].disks.include` | `` (*list of strings*) | No | Only include the disks with the given name; case-sensitive; regular expressions are supported. | -| `environments[$env].disks.exclude` | `` (*list of strings*) | No | Only include the disks with the given name; case-sensitive; regular expressions are supported. | -| `environments[$env].disks.all_others` | `include` (*one of `include`, `exclude`*) | No | Behaviour for disks which are not explicitly included or excluded. | +| Key | Default | Required | Description | +|---------------------------------------------|-------------------------------------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `port` | `80` (*int*) | No | Default HTTP port to listen for requests. TLS is not supported at the moment. Consider using a proxy if you need encryption. | +| `update_interval` | `1h` (*duration*) | No | Checks each disk in that duration interval. [time.ParseDuration format](https://pkg.go.dev/time#example-ParseDuration) must be used. | +| `log_level` | `` (*one of `debug`, `info`*) | No | Used log level; will be overwritten if `--debug` is used. | +| `downloads.enabled` | `false` | No | If `true`, the latest artifact of a monitored backup disk can be downloaded. This is disabled by default for security reasons ([#1](https://github.com/dreitier/backmon/issues/1)). | +| `http.basic_auth.username` | `` (*string*) | No | Username for HTTP Basic Authentication. If this is set, `http.basic_auth.password` must be also set. | +| `http.basic_auth.password` | `` (*string*) | No | Password for HTTP Basic Authentication. If this is set, `http.basic_auth.username` must be also set. | +| `http.tls.certificate` | `` (*string*) | No | Path to certificate file. If this is set, `http.tls.key` must be also set. | +| `http.tls.key` | `` (*string*) | No | Path to private key file. If this is set, `http.tls.certificate` must be also set. | +| `http.tls.strict` | `false` (*bool*) | No | If set to true, a preferred TLS default configuration is used. | +| `environments` | `` (*list of environment*) | No | Each `environment` to check. | +| `environments[]` | `` (*string*) | __Yes__ | Name of environment. | +| `environments[$env].definitions` | `backup_definitions.yaml` (*string*) | No | YAML file containing the backup definitions. | +| `environments[$env].path` | `` (*string*) | No | Local path to check for. If you use the `path` parameter, other parameters specific for S3 are ignored. | +| `environments[$env].s3.region` | `eu-central-1` (*string*) | No | AWS region | +| `environments[$env].s3.access_key_id` | `` (*string*) | __Yes__ | AWS Access Key | +| `environments[$env].s3.secret_access_key` | `` (*string*) | __Yes__ | AWS Secret Access Key | +| `environments[$env].s3.endpoint` | `` (*string*) | No | Custom AWS S3 endpoint. This must be used for Minio buckets or if you are using a local S3 instance. | +| `environments[$env].s3.force_path_style` | `false` (*bool*) | No | Use path-style for that S3 bucket. This is deprecated by AWS S3 and should be probably `false`. | +| `environments[$env].s3.insecure` | `false` (*bool*) | No | Whether to use HTTP or HTTPS to access the S3 endpoint. This is intended primarily for accessing a local S3 endpoint like MinIO. | +| `environments[$env].s3.tls_skip_verify` | `false` (*bool*) | No | Whether to verify the X.509 certificated presented by the S3 endpoint. This is intended primarily for accessing a local S3 endpoint which uses a self signed certificate | +| `environments[$env].s3.token` | `` (*string*) | No | AWS STS session token. You can leave that empty. | +| `environments[$env].s3.auto_discover_disks` | `true` (*bool*) | No | Automatically iterate over each S3 bucket. | +| `environments[$env].disks.include` | `` (*list of strings*) | No | Only include the disks with the given name; case-sensitive; regular expressions are supported. | +| `environments[$env].disks.exclude` | `` (*list of strings*) | No | Only include the disks with the given name; case-sensitive; regular expressions are supported. | +| `environments[$env].disks.all_others` | `include` (*one of `include`, `exclude`*) | No | Behaviour for disks which are not explicitly included or excluded. | ## `disks` @@ -121,6 +124,6 @@ disks: ``` :::info -If you have set `environments[$env].auto_discover_disks` to `false`, only the `disks.include` configuration parameter +If you have set `environments[$env].s3.auto_discover_disks` to `false`, only the `disks.include` configuration parameter makes any sense. ::: diff --git a/docs/20-reference/10-backmon-configuration/10-storage/20-aws-s3.md b/docs/20-reference/10-backmon-configuration/10-storage/20-aws-s3.md index 499b215..ba50b35 100755 --- a/docs/20-reference/10-backmon-configuration/10-storage/20-aws-s3.md +++ b/docs/20-reference/10-backmon-configuration/10-storage/20-aws-s3.md @@ -10,6 +10,8 @@ environments: secret_access_key: my_secret_access_key ``` +For a full list of configuration option for the S3 provider see [`the corresponding section`](../01-overview.md#available-configuration-keys) + Your AWS account contains the following S3 buckets: - `bucket-1` @@ -158,4 +160,4 @@ backmon: - key: BACKMON_AWS_SECRET_ACCESS_KEY # replace ${SECRET_ACCESS_KEY} with the secret access key you noted down previously value: ${SECRET_ACCESS_KEY} -``` \ No newline at end of file +``` diff --git a/docs/20-reference/20-backup-definition/10-overview.md b/docs/20-reference/20-backup-definition/10-overview.md index 1e023a4..9e35f6f 100755 --- a/docs/20-reference/20-backup-definition/10-overview.md +++ b/docs/20-reference/20-backup-definition/10-overview.md @@ -6,87 +6,97 @@ import TabItem from '@theme/TabItem'; The backup definition is a YAML file that describes the structure of your backups. In it, you provide the information that *backmon* needs to collect metric data, purge files, and offer file downloads. -*backmon* looks for the backup definition file called `backup_definitions.yaml` at the root directory of your disk. +*backmon* looks for the backup definition file called `backup_definitions.yaml` at the root directory of your disk. You can overwrite the file naming in the [backmon configuration file](../10-backmon-configuration/01-overview.md). ### Sample `backup_definitions.yaml` file ```yaml -# use directory ./databases as the the root directory. You can have multiple directories in each YAML file. -# use '.' if the definitions file is in the same directory as your backups. -'./databases/{{client}}': - # alias is used for Prometheus metrics - alias: DB - # assign variables - fuse: - - 'client' - # apply those defaults to all files.* sections. You can override each parameter in the files.* section - defaults: - # expect that a backup file is generated at 00:30 each night - schedule: 30 0 * * * - # for identifying the newest file, we use the `born_at` field - sort: born_at - # purge old files which are older than 7 days and keep at least 10 backups - purge: true - - # @see https://github.com/dreitier/backmon/issues/19 - # retention-count: 10 - # retention-age: 7d - # inside the ./databases folder there are multiple files like `postgres-20220713.gz` - files: - 'postgres-%Y%M%D.gz': - # File alias for Prometheus - alias: PostgeSQL - # This backup is generated at 01:00 each night - schedule: 0 1 * * * - retention-count: 10 - retention-age: 7d - '%Y%M%D_mysql_%I.gz': - alias: MySQL +quota: 2GiB +directories: + # use directory ./databases as the the root directory. You can have multiple directories in each YAML file. + # use '.' if the definitions file is in the same directory as your backups. + './databases/{{client}}': + # alias is used for Prometheus metrics + alias: DB + # assign variables + fuse: + - 'client' + # apply those defaults to all files.* sections. You can override each parameter in the files.* section + defaults: + # expect that a backup file is generated at 00:30 each night + schedule: 30 0 * * * + # for identifying the newest file, we use the `born_at` field + sort: born_at + # purge old files which are older than 7 days and keep at least 10 backups + purge: true + + # @see https://github.com/dreitier/backmon/issues/19 + # retention-count: 10 + # retention-age: 7d + # inside the ./databases folder there are multiple files like `postgres-20220713.gz` + files: + 'postgres-%Y%M%D.gz': + # File alias for Prometheus + alias: PostgeSQL + # This backup is generated at 01:00 each night + schedule: 0 1 * * * + retention-count: 10 + retention-age: 7d + '%Y%M%D_mysql_%I.gz': + alias: MySQL ``` ## Available configuration keys -| Key | Default | Required | Description | -| --- | --- | --- | --- | -| `[$directory]` | `` (*string*) | __Yes__ | Directory name inside the disk, relative to the `backup_definitions.yaml`. | -| `[$directory].alias` | `` (*string*) | No | Used alias when exporting metrics. | -| `[$directory].fuse[]` | `` (*list of wildcard strings*) | No | Group files together by that substitutions. | -| `[$directory].defaults.*` | `` (*any of .schedule, .sort, .purge*) | No | Apply those defaults for each `[$directory].files[$file]` section. Each `[$directory].files[$file].*` key can override this default.

(`.retention-count`, `.retention-age`) are not available, see [#19](https://github.com/dreitier/backmon/issues/19). | -| `[$directory].files[$file]` | `` (*map*) | __Yes__ | File pattern to check for.| -| `[$directory].files[$file].schedule` | `` (*valid cron definition*) | No | Cron definition to check for files | -| `[$directory].files[$file].sort` | `interpolation` (*one of `born_at`, `modified_at`, `archived_at` or `interpolation`*) | No | Used data to sort the files. The selected method uses the given field to sort the files and find the latest/newest one. In case of `interpolation`, the variable substitution is applied | -| `[$directory].files[$file].purge` | `` (*bool*) | No | Purges each file which does not match `retention-age` or `retention-count`. If `purge` is set to `true`, without having an explicit retention, a limit von `14` days is assumed. | -| `[$directory].files[$file].retention-count` | `` (*int*) | No | Keep max number of files | -| `[$directory].files[$file].retention-age` | `` (*duration*) | No | Keep files newer than that | -| `[$directory].files[$file].alias` | `` (*string*) | No | Used alias when exporting metrics. | - - -## `[$directory]` +| Key | Default | Required | Description | +|---------------------------------------------------------|---------------------------------------------------------------------------------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `quota` | `0B` (*size*) | No | Bucket quota. Supported units are for example GB (10^n) or GiB (2^n) | +| `directories.[$directory]` | `` (*string*) | __Yes__ | Directory name inside the disk, relative to the `backup_definitions.yaml`. | +| `directories.[$directory].alias` | `` (*string*) | No | Used alias when exporting metrics. | +| `directories.[$directory].fuse[]` | `` (*list of wildcard strings*) | No | Group files together by that substitutions. | +| `directories.[$directory].defaults.*` | `` (*any of .schedule, .sort, .purge*) | No | Apply those defaults for each `directories.[$directory].files[$file]` section. Each `directories.[$directory].files[$file].*` key can override this default.

(`.retention-count`, `.retention-age`) are not available, see [#19](https://github.com/dreitier/backmon/issues/19). | +| `directories.[$directory].files[$file]` | `` (*map*) | __Yes__ | File pattern to check for. | +| `directories.[$directory].files[$file].schedule` | `` (*valid cron definition*) | No | Cron definition to check for files | +| `directories.[$directory].files[$file].sort` | `interpolation` (*one of `born_at`, `modified_at`, `archived_at` or `interpolation`*) | No | Used data to sort the files. The selected method uses the given field to sort the files and find the latest/newest one. In case of `interpolation`, the variable substitution is applied | +| `directories.[$directory].files[$file].purge` | `` (*bool*) | No | Purges each file which does not match `retention-age` or `retention-count`. If `purge` is set to `true`, without having an explicit retention, a limit von `14` days is assumed. | +| `directories.[$directory].files[$file].retention-count` | `` (*int*) | No | Keep max number of files | +| `directories.[$directory].files[$file].retention-age` | `` (*duration*) | No | Keep files newer than that | +| `directories.[$directory].files[$file].alias` | `` (*string*) | No | Used alias when exporting metrics. | + +## `directories.[$directory]` + The root elements for each `backup_definitions.yaml` are the names of the subdirectories in a disk. ### Location of backup directories + #### Backups in the same directory + When backups are in the same directory with the `backup_definitions.yaml`, you have to use `.` or `./` as root element: ```mdx-code-block ``` + ```bash - backup_definitions.yaml - backup1.tar.gz - backup2.tar.gz ``` + ```mdx-code-block ``` + ```yaml -'./': - alias: root - # ... configuration options +directories: + './': + alias: root + # ... configuration options ``` + ```mdx-code-block @@ -98,6 +108,7 @@ When backups are in the same directory with the `backup_definitions.yaml`, you h ``` + ```bash - backup_definitions.yaml + /backup-1/ @@ -107,18 +118,22 @@ When backups are in the same directory with the `backup_definitions.yaml`, you h - backup1.tar.gz - backup2.tar.gz ``` + ```mdx-code-block ``` + ```yaml -'./backup-1': - alias: root-1 - # ... configuration options -'./backup-2': - alias: root-2 - # ... configuration options +directories: + './backup-1': + alias: root-1 + # ... configuration options + './backup-2': + alias: root-2 + # ... configuration options ``` + ```mdx-code-block @@ -127,7 +142,10 @@ When backups are in the same directory with the `backup_definitions.yaml`, you h ### Patterns #### Pattern variables -A variable can be put on a *directory* pattern by using two curly braces (e.g. `{{variable}}`). For variable names only characters `0-9`, `A-Z`, `a-z`, and `_` are allowed. Please note that you _should not_ override the variables names from the [substituions](substitutions). + +A variable can be put on a *directory* pattern by using two curly braces (e.g. `{{variable}}`). For variable names only +characters `0-9`, `A-Z`, `a-z`, and `_` are allowed. Please note that you _should not_ override the variables names from +the [substituions](substitutions). Variables can be referenced later on in a *file* pattern with help of `${variable}`. @@ -135,63 +153,78 @@ Variables can be referenced later on in a *file* pattern with help of `${variabl See [substituions](substitutions). -### `[$directory].alias` +### `directories.[$directory].alias` + If specified, the label `dir` for the epxorted Prometheues metrics will be overwritten with the given alias. #### Without `.alias` + ```mdx-code-block ``` + ```yaml -'./backups': - files: - 'dump-%Y%M%D.sql': +directories: + './backups': + files: + 'dump-%Y%M%D.sql': ``` + ```mdx-code-block ``` + ```bash -backmon_backup_file_age_aim_seconds{dir="./backups",disk="_samples/1.postgres-dumps",file="dump-%Y%M%D.sql"} +backmon_backup_latest_file_creation_expected_at_timestamp_seconds{dir="./backups",disk="_samples/1.postgres-dumps",file="dump-%Y%M%D.sql"} ``` + ```mdx-code-block ``` #### With `.alias` specified + ```mdx-code-block ``` + ```yaml -'./backups': - alias: "custom_backups" - files: - 'dump-%Y%M%D.sql': +directories: + './backups': + alias: "custom_backups" + files: + 'dump-%Y%M%D.sql': ``` + ```mdx-code-block ``` + ```bash -backmon_backup_file_age_aim_seconds{dir="custom_backups",disk="_samples/1.postgres-dumps",file="dump-%Y%M%D.sql"} +backmon_backup_latest_file_creation_expected_at_timestamp_seconds{dir="custom_backups",disk="_samples/1.postgres-dumps",file="dump-%Y%M%D.sql"} ``` + ```mdx-code-block ``` +### `directories.[$directory].fuse` -### `[$directory].fuse` -Sometimes you need to group files together which belong logically to the same backup set but are split across different directories. +Sometimes you need to group files together which belong logically to the same backup set but are split across different +directories. ```mdx-code-block Let's group together the following backup `.sql` files: ``` + ```bash backups/2022/01/postgres/some_subdir - 20220101.sql @@ -200,46 +233,59 @@ backups/2022/02/postgres/some_subdir backups/2022/03/postgres/some_subdir - 20220301.sql ``` + ```mdx-code-block ``` + ```yaml -'./backups/{{of_year}/{{of_month}}/postgres/{{some_subdir}}': - alias: "backups" - fuse: - - 'of_year' - - 'of_month' - files: - 'dump-%Y%M%D.sql': - alias: 'postgres' -``` +directories: + './backups/{{of_year}/{{of_month}}/postgres/{{some_subdir}}': + alias: "backups" + fuse: + - 'of_year' + - 'of_month' + files: + 'dump-%Y%M%D.sql': + alias: 'postgres' + ``` + ```mdx-code-block ``` -### `[$directory].defaults.*` -If you want to use the same configuration over multiple `[$directory].files[$file]` sections, you can use `.defaults`. -With `.defaults` you can apply the configuration keys +### `directories.[$directory].defaults.*` + +If you want to use the same configuration over multiple `directories.[$directory].files[$file]` sections, you can use `.defaults`. +With `.defaults` you can apply the configuration keys + - `.schedule` - `.sort` - `.purge` -as defaults for each `[$directory].files[$file].*` section. You can can override the default key value in each `[$directory].files[$file]` section. +as defaults for each `directories.[$directory].files[$file].*` section. You can can override the default key value in each +`directories.[$directory].files[$file]` section. The two settings + - `.retention-count` - `.retention-age` are **not** available, see [#19](https://github.com/dreitier/backmon/issues/19). -### `[$directory].files[$file]` -Each `[$directory].files[$file]` key supports so called [substitutions](substitutions). They are basically simplified regular expressions. Some of those substitutions like '%Y' and '%M' are used to update the file's `interpolated_timestamp'. + +### `directories.[$directory].files[$file]` + +Each `directories.[$directory].files[$file]` key supports so-called [substitutions](substitutions). They are basically simplified +regular expressions. Some of those substitutions like '%Y' and '%M' are used to update the file's ` +interpolated_timestamp'. ```mdx-code-block ``` + ```bash backups-1/ - dump-20220401.sql @@ -247,90 +293,115 @@ backups-1/ - dump-20220601.sql - dump-20220701.sql ``` + ```mdx-code-block ``` + ```yaml -./backups-1: - alias: "backups" - files: - 'dump-%Y%M%D.sql': +directories: + ./backups-1: + alias: "backups" + files: + 'dump-%Y%M%D.sql': ``` + ```mdx-code-block ``` +### `directories.[$directory].files[$file].schedule` -### `[$directory].files[$file].schedule` Put in your crontab definition for when the backup should occur. -### `[$directory].files[$file].sort` -With help of `.sort` you can define which of the backup file's fields is used for sorting. Sorting is used for finding the latest/newest files (relevant for the `/api/download` endpoint) and oldest files (relevant for purging). +### `directories.[$directory].files[$file].sort` + +With help of `.sort` you can define which of the backup file's fields is used for sorting. Sorting is used for finding +the latest/newest files (relevant for the `/api/download` endpoint) and oldest files (relevant for purging). + +| Key | Timestamp used | Default | +|-----------------|---------------------------------------------------------|---------| +| `born_at` | [`born_at`](file-dates) | - | +| `modified_at` | [`modified_at`](file-dates) | - | +| `archived_at` | [`archived_at`](file-dates) | - | +| `interpolation` | [interpolated timestamp](file-dates#interpolated_value) | ✓ | -| Key | Timestamp used | Default | -| --- | --- | --- | -| `born_at` | [`born_at`](file-dates) | - | -| `modified_at` | [`modified_at`](file-dates) | - | -| `archived_at` | [`archived_at`](file-dates) | - | -| `interpolation` | [interpolated timestamp](file-dates#interpolated_value) | ✓ | +If you are not using [.stat / dotstat files](file-dates#stat--dotstat-files), [ +`interpolation`](file-dates#interpolated_timestamp) is used. -If you are not using [.stat / dotstat files](file-dates#stat--dotstat-files), [`interpolation`](file-dates#interpolated_timestamp) is used. +### `directories.[$directory].files[$file].purge` -### `[$directory].files[$file].purge` If set to `true`, files which are older than `retention-age` and over the number of `retention-count` are deleted. -### `[$directory].files[$file].retention-count` -If `.purge` is set to true, at least a number of `.retention-count` files are kept, even if they are older than `.retention-age`. +### `directories.[$directory].files[$file].retention-count` -### `[$directory].files[$file].retention-age` -If `.purge` is set to true, files older then `.retention-age` are purged. Pleas note that at least a number of `.retention-count` files is kept. +If `.purge` is set to true, at least a number of `.retention-count` files are kept, even if they are older than +`.retention-age`. + +### `directories.[$directory].files[$file].retention-age` + +If `.purge` is set to true, files older then `.retention-age` are purged. Pleas note that at least a number of +`.retention-count` files is kept. `.sort` is used to calculate if a given file is older than `.retention-age` from today. -### `[$directory].files[$file].alias` +### `directories.[$directory].files[$file].alias` + If specified, the label `file` for the epxorted Prometheues metrics will be overwritten with the given alias. #### Without `.alias` + ```mdx-code-block ``` + ```yaml -'./backups': - files: - 'dump-%Y%M%D.sql': +directories: + './backups': + files: + 'dump-%Y%M%D.sql': ``` + ```mdx-code-block ``` + ```bash -backmon_backup_file_age_aim_seconds{dir="./backups",disk="_samples/1.postgres-dumps",file="dump-%Y%M%D.sql"} +backmon_backup_latest_file_creation_expected_at_timestamp_seconds{dir="./backups",disk="_samples/1.postgres-dumps",file="dump-%Y%M%D.sql"} ``` + ```mdx-code-block ``` #### With `.alias` specified + ```mdx-code-block ``` + ```yaml -'./backups': - files: - 'dump-%Y%M%D.sql': - alias: "pg-dump" +directories: + './backups': + files: + 'dump-%Y%M%D.sql': + alias: "pg-dump" ``` + ```mdx-code-block ``` + ```bash -backmon_backup_file_age_aim_seconds{dir="./backups",disk="_samples/1.postgres-dumps",file="pg-dump"} +backmon_backup_latest_file_creation_expected_at_timestamp_seconds{dir="./backups",disk="_samples/1.postgres-dumps",file="pg-dump"} ``` + ```mdx-code-block diff --git a/docs/20-reference/20-backup-definition/100-how-tos/10-rancher-backups.md b/docs/20-reference/20-backup-definition/100-how-tos/10-rancher-backups.md index f01fc12..f4cbdd0 100755 --- a/docs/20-reference/20-backup-definition/100-how-tos/10-rancher-backups.md +++ b/docs/20-reference/20-backup-definition/100-how-tos/10-rancher-backups.md @@ -12,13 +12,14 @@ s3-backup-b025385f-800a-4ee6-b8b6-bbf199a0e733-2022-08-04T00-00-00Z.tar.gz To monitor those files, create a `backup_definitions.yaml` in the S3 bucket with the following content: ```yaml -content: - alias: RANCHER_CLUSTER - defaults: - schedule: 0 0 * * * - files: - "s3-backup-%w-%w-%w-%w-%w-%Y-%M-%DT%h-%m-%sZ.tar.gz": - alias: rancher-backups - retention-count: 7 - purge: false -``` \ No newline at end of file +directories: + content: + alias: RANCHER_CLUSTER + defaults: + schedule: 0 0 * * * + files: + "s3-backup-%w-%w-%w-%w-%w-%Y-%M-%DT%h-%m-%sZ.tar.gz": + alias: rancher-backups + retention-count: 7 + purge: false +``` diff --git a/docs/20-reference/metrics.md b/docs/20-reference/metrics.md index 807694f..6e1f7f4 100755 --- a/docs/20-reference/metrics.md +++ b/docs/20-reference/metrics.md @@ -2,107 +2,136 @@ One of the main tasks of *backmon* is to gather and export metrics. Here you will find descriptions of: + - the [metric gathering process](#metric-data-collection) - the [Prometheus endpoint](#prometheus-endpoint), and - the [exported metrics](#available-metrics) ## Metric Data Collection - - ## Prometheus Endpoint + Metrics are offered through a configurable Prometheus endpoint. The prometheus endpoint is reachable on the *backmon* host under the `/metrics` path. - ## Available Metrics -All of the metrics offered by *backmon* begin with the prefix `backmon_backup_`. -The metrics are listed under their full names here, but other parts of the documentation may omit this prefix for the sake of brevity. + +All of the metrics offered by *backmon* begin with the prefix `backmon_backup_`. +The metrics are listed under their full names here, but other parts of the documentation may omit this prefix for the +sake of brevity. ### `backmon_environments_total` + Total number of environments, configured in the `config.yaml`. ### `backmon_disks_total` + Total number of discovered disks. This includes also the number of inactive disks, e.g. if permissions are missing. -### `backmon_backup_status` -This is a special metric that reports on the status of a bucket. +### `backmon_definition_status` + +This is a special metric that reports on the status of a definitions file. -When there are no issues with the bucket, the reported status will be `0`. -A `0` means that *backmon* successfully connected to the bucket, retrieved the [*backup-definition*](20-backup-definition/10-overview.md) file, and parsed it. +When the definitions file could be accessed and parsed correctly, the reported status will be `1`. +A `1` means that *backmon* successfully connected to the bucket/disk, retrieved the [ +*backup-definition*](20-backup-definition/10-overview.md) file, and parsed it. -If there is an issue with the bucket, the reported status will be an error code. -Such an error code will always be an integer > `0` from the following list: +If there is an issue with the bucket, the reported status will be an error code from the following list: | Status | Meaning | |-------:|:--------------| -| `0` | OK | -| `1` | Unknown error | +| `0` | Unknown error | +| `1` | OK | + + +### `backmon_file_count_total` + +Reports the total number of files in a bucket regardless of whether they are part of a backup definition or not. + +### `backmon_disk_usage_bytes` + +Reports the total amount of storage used by all files in a bucket regardless of whether they are part of a backup definition or not. -> **TBD**: error codes +### `backmon_backup_file_count_max` -### `backmon_backup_file_count_aim` Reports the desired file count, as specified by the `retention-count` setting in the *backup_definitions.yaml*. ### `backmon_backup_file_count` + Reports the actual file count. -When purging is enabled, this count will usually be equal to the [target](#backmon_backup_file_count_aim). +When purging is enabled, this count will usually be equal to the [target](#backmon_backup_file_count_max). + +### `backmon_backup_file_age_max_seconds` -### `backmon_backup_file_age_aim_seconds` -This is equal to the converted `retention-age` parameter in seconds from the [`backup_definitions.yaml`](backup-definition/overview) +This is equal to the converted `retention-age` parameter in seconds from the [ +`backup_definitions.yaml`](backup-definition/overview) ### `backmon_backup_file_young_count` + The amount of backup files in this group that are younger than the maximum age (`file_age_aim_seconds`). -### `backmon_backup_latest_creation_aim_seconds` +### `backmon_backup_file_creation_expected_at` + Unix timestamp on which the latest backup in the corresponding file group should have occurred. -### `backmon_backup_latest_creation_seconds` +### `backmon_backup_latest_file_created_at_timestamp_seconds` + Unix timestamp on which the latest backup in the corresponding file group was created. This is basically the value which comes from the variable interpolation of the file. ### `backmon_backup_latest_file_creation_duration` -Describes how long it took to create the backup file in seconds. This is 0 if born_at and modified_at have the same value. -### `backmon_backup_latest_file_born_at` +Describes how long it took to create the backup file in seconds. This is 0 if born_at and modified_at have the same +value. + +### `backmon_backup_latest_file_born_at_timestamp_seconds` + Unix timestamp on which the latest file has been initially created. See [.stat files](backup-definition/file-dates). -### `backmon_backup_latest_file_modified_at` +### `backmon_backup_latest_file_modified_at_timestamp_seconds` + Unix timestamp on which the latest file has been modified. See [.stat files](backup-definition/file-dates). -### `backmon_backup_latest_file_archived_at` +### `backmon_backup_latest_file_archived_at_timestamp_seconds` + Unix timestamp on which the latest file has been archived. See [.stat files](backup-definition/file-dates). ### `backmon_backup_latest_size_bytes` + Size (in bytes) of the latest backup in the corresponding file group. ## A note about `backmon_backup_latest_file_*_at` -Due to [file system limitations](backup-definition/file-dates) section, you have to provide the values for -- `backmon_backup_latest_file_born_at` -- `backmon_backup_latest_file_modified_at` -- `backmon_backup_latest_file_archived_at` -through a [.stat file](backup-definition/file-dates#stat--dotstat-files). If you do not that, each of the metrics will have the same value/timestamp. +Due to [file system limitations](backup-definition/file-dates) section, you have to provide the values for + +- `backmon_backup_latest_file_born_at_timestamp_seconds` +- `backmon_backup_latest_file_modified_at_timestamp_seconds` +- `backmon_backup_latest_file_archived_at_timestamp_seconds` + +through a [.stat file](backup-definition/file-dates#stat--dotstat-files). If you do not that, each of the metrics will +have the same value/timestamp. ## Labels + The following table shows which labels are defined for each metric. -| Metric | disk | dir | file | group | -|:---------------------------------------------|:----:|:---:|:----:|:-----:| -| `backmon_backup_status` | ✓ | - | - | - | -| `backmon_backup_file_count_aim` | ✓ | ✓ | ✓ | - | -| `backmon_backup_file_count` | ✓ | ✓ | ✓ | ✓ | -| `backmon_backup_file_age_aim_seconds` | ✓ | ✓ | ✓ | - | -| `backmon_backup_file_young_count` | ✓ | ✓ | ✓ | ✓ | -| `backmon_backup_latest_creation_aim_seconds` | ✓ | ✓ | ✓ | - | -| `backmon_backup_latest_creation_seconds` | ✓ | ✓ | ✓ | ✓ | -| `backmon_backup_latest_creation_duration` | ✓ | ✓ | ✓ | ✓ | -| `backmon_backup_latest_file_born_at` | ✓ | ✓ | ✓ | ✓ | -| `backmon_backup_latest_file_modified_at` | ✓ | ✓ | ✓ | ✓ | -| `backmon_backup_latest_file_archived_at` | ✓ | ✓ | ✓ | ✓ | -| `backmon_backup_latest_creation_seconds` | ✓ | ✓ | ✓ | ✓ | -| `backmon_backup_latest_size_bytes` | ✓ | ✓ | ✓ | ✓ | +| Metric | disk | dir | file | group | +|:-----------------------------------------------------------------------|:----:|:---:|:----:|:-----:| +| `backmon_definition_status` | ✓ | - | - | - | +| `backmon_disk_usage_bytes` | ✓ | - | - | - | +| `backmon_disk_quota_bytes` | ✓ | - | - | - | +| `backmon_backup_file_count_max` | ✓ | ✓ | ✓ | - | +| `backmon_backup_file_count` | ✓ | ✓ | ✓ | ✓ | +| `backmon_file_count_total` | ✓ | ✓ | ✓ | ✓ | +| `backmon_backup_file_age_max_seconds` | ✓ | ✓ | ✓ | - | +| `backmon_backup_file_young_count` | ✓ | ✓ | ✓ | ✓ | +| `backmon_backup_latest_file_creation_expected_at_timestamp_seconds` | ✓ | ✓ | ✓ | - | +| `backmon_backup_latest_file_creation_duration` | ✓ | ✓ | ✓ | ✓ | +| `backmon_backup_latest_file_born_at_timestamp_seconds` | ✓ | ✓ | ✓ | ✓ | +| `backmon_backup_latest_file_modified_at_timestamp_seconds` | ✓ | ✓ | ✓ | ✓ | +| `backmon_backup_latest_file_archived_at_timestamp_seconds` | ✓ | ✓ | ✓ | ✓ | +| `backmon_backup_latest_size_bytes` | ✓ | ✓ | ✓ | ✓ | ## Sample output from the `/api/metrics` endpoint @@ -113,40 +142,40 @@ backmon_disks_total 0 # HELP backmon_environments_total Total number of environments # TYPE backmon_environments_total gauge backmon_environments_total 1 -# HELP backmon_backup_file_age_aim_seconds The maximum age (in seconds) that any file in this group should reach. -# TYPE backmon_backup_file_age_aim_seconds gauge -backmon_backup_file_age_aim_seconds{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump"} 0 +# HELP backmon_backup_file_age_max_seconds The maximum age (in seconds) that any file in this group should reach. +# TYPE backmon_backup_file_age_max_seconds gauge +backmon_backup_file_age_max_seconds{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump"} 0 # HELP backmon_backup_file_count The amount of backup files present in this group. # TYPE backmon_backup_file_count gauge backmon_backup_file_count{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump",group="backups"} 4 -# HELP backmon_backup_file_count_aim The amount of backup files expected to be present in this group. -# TYPE backmon_backup_file_count_aim gauge -backmon_backup_file_count_aim{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump"} 0 +# HELP backmon_backup_file_count_max The amount of backup files expected to be present in this group. +# TYPE backmon_backup_file_count_max gauge +backmon_backup_file_count_max{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump"} 0 # HELP backmon_backup_file_young_count The amount of backup files in this group that are younger than the maximum age (file_age_aim_seconds). # TYPE backmon_backup_file_young_count gauge backmon_backup_file_young_count{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump",group="backups"} 0 -# HELP backmon_backup_latest_creation_aim_seconds Unix timestamp on which the latest backup in the corresponding file group should have occurred. -# TYPE backmon_backup_latest_creation_aim_seconds gauge -backmon_backup_latest_creation_aim_seconds{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump"} 1.6581852e+09 -# HELP backmon_backup_latest_creation_seconds Unix timestamp on which the latest backup in the corresponding file group was created. -# TYPE backmon_backup_latest_creation_seconds gauge -backmon_backup_latest_creation_seconds{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump",group="backups"} 1.658182599e+09 -# HELP backmon_backup_latest_file_archived_at Unix timestamp on which the latest file has been archived -# TYPE backmon_backup_latest_file_archived_at gauge -backmon_backup_latest_file_archived_at{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump",group="backups"} 1.381818188e+09 -# HELP backmon_backup_latest_file_born_at Unix timestamp on which the latest file has been initially created -# TYPE backmon_backup_latest_file_born_at gauge -backmon_backup_latest_file_born_at{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump",group="backups"} 1.2313213e+07 +# HELP backmon_backup_latest_file_creation_expected_at_timestamp_seconds Unix timestamp on which the latest backup in the corresponding file group should have occurred. +# TYPE backmon_backup_latest_file_creation_expected_at_timestamp_seconds gauge +backmon_backup_latest_file_creation_expected_at_timestamp_seconds{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump"} 1.6581852e+09 +# HELP backmon_backup_latest_file_created_at_timestamp_seconds Unix timestamp on which the latest backup in the corresponding file group was created. +# TYPE backmon_backup_latest_file_created_at_timestamp_seconds gauge +backmon_backup_latest_file_created_at_timestamp_seconds{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump",group="backups"} 1.658182599e+09 +# HELP backmon_backup_latest_file_archived_at_timestamp_seconds Unix timestamp on which the latest file has been archived +# TYPE backmon_backup_latest_file_archived_at_timestamp_seconds gauge +backmon_backup_latest_file_archived_at_timestamp_seconds{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump",group="backups"} 1.381818188e+09 +# HELP backmon_backup_latest_file_born_at_timestamp_seconds Unix timestamp on which the latest file has been initially created +# TYPE backmon_backup_latest_file_born_at_timestamp_seconds gauge +backmon_backup_latest_file_born_at_timestamp_seconds{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump",group="backups"} 1.2313213e+07 # HELP backmon_backup_latest_file_creation_duration Describes how long it took to create the backup file in seconds # TYPE backmon_backup_latest_file_creation_duration gauge backmon_backup_latest_file_creation_duration{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump",group="backups"} 1.17626186e+08 -# HELP backmon_backup_latest_file_modified_at Unix timestamp on which the latest file has been modified -# TYPE backmon_backup_latest_file_modified_at gauge -backmon_backup_latest_file_modified_at{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump",group="backups"} 1.29939399e+08 +# HELP backmon_backup_latest_file_modified_at_timestamp_seconds Unix timestamp on which the latest file has been modified +# TYPE backmon_backup_latest_file_modified_at_timestamp_seconds gauge +backmon_backup_latest_file_modified_at_timestamp_seconds{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump",group="backups"} 1.29939399e+08 # HELP backmon_backup_latest_size_bytes Size (in bytes) of the latest backup in the corresponding file group. # TYPE backmon_backup_latest_size_bytes gauge backmon_backup_latest_size_bytes{dir="postgres",disk="samples/1.postgres-dumps",file="pgdump",group="backups"} 0 -# HELP backmon_backup_status Indicates whether there were any problems collecting metrics for this disk. Any value >0 means that errors occurred. -# TYPE backmon_backup_status gauge -backmon_backup_status{disk="samples/1.postgres-dumps"} 0 +# HELP backmon_definition_status Indicates whether there were any problems collecting metrics for this disk. Any value >0 means that errors occurred. +# TYPE backmon_definition_status gauge +backmon_definition_status{disk="samples/1.postgres-dumps"} 1 ``` \ No newline at end of file diff --git a/docs/30-faq.md b/docs/30-faq.md index 09f7628..c06d611 100755 --- a/docs/30-faq.md +++ b/docs/30-faq.md @@ -9,4 +9,4 @@ sum_over_time(min(1, irate(backup_latest_time_seconds[7d]))) ### Why does Prometheus not export metrics like `backmon_backup_latest_size_bytes`? -If metrics like `backmon_backup_latest_size_bytes` and `backmon_backup_latest_file_modified_at` are not available in the `/metrics` endpoint, either the root directories name in your `backup_definitions.yaml` is wrong or none of the files matches the given pattern. \ No newline at end of file +If metrics like `backmon_backup_latest_size_bytes` and `backmon_backup_latest_file_modified_at_timestamp_seconds` are not available in the `/metrics` endpoint, either the root directories name in your `backup_definitions.yaml` is wrong or none of the files matches the given pattern. \ No newline at end of file