mirror of https://github.com/gogs/gogs.git
docker: add scheduled backups with retention policy (#6140)
Aleksandar Puharic
GitHub
parent
1125bb5848
commit
2003864615
|
|
@ -13,6 +13,7 @@ All notable changes to Gogs are documented in this file.
|
|||
- Able to fill in pull request title with a template. [#5901](https://github.com/gogs/gogs/pull/5901)
|
||||
- Able to override static files under `public/` directory, please refer to [documentation](https://gogs.io/docs/features/custom_template) for usage. [#5920](https://github.com/gogs/gogs/pull/5920)
|
||||
- New API endpoint `GET /admin/teams/:teamid/members` to list members of a team. [#5877](https://github.com/gogs/gogs/issues/5877)
|
||||
- Support backup with retention policy for Docker deployments. [#6140](https://github.com/gogs/gogs/pull/6140)
|
||||
|
||||
### Changed
|
||||
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ COPY --from=binarybuilder /gogs.io/gogs/gogs .
|
|||
RUN ./docker/finalize.sh
|
||||
|
||||
# Configure Docker Container
|
||||
VOLUME ["/data"]
|
||||
VOLUME ["/data", "/backup"]
|
||||
EXPOSE 22 3000
|
||||
ENTRYPOINT ["/app/gogs/docker/start.sh"]
|
||||
CMD ["/bin/s6-svscan", "/app/gogs/docker/s6/"]
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ COPY --from=binarybuilder /gogs.io/gogs/gogs .
|
|||
RUN ./docker/finalize.sh
|
||||
|
||||
# Configure Docker Container
|
||||
VOLUME ["/data"]
|
||||
VOLUME ["/data", "/backup"]
|
||||
EXPOSE 22 3000
|
||||
ENTRYPOINT ["/app/gogs/docker/start.sh"]
|
||||
CMD ["/bin/s6-svscan", "/app/gogs/docker/s6/"]
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ COPY --from=binarybuilder /gogs.io/gogs/gogs .
|
|||
RUN ./docker/finalize.sh
|
||||
|
||||
# Configure Docker Container
|
||||
VOLUME ["/data"]
|
||||
VOLUME ["/data", "/backup"]
|
||||
EXPOSE 22 3000
|
||||
ENTRYPOINT ["/app/gogs/docker/start.sh"]
|
||||
CMD ["/bin/s6-svscan", "/app/gogs/docker/s6/"]
|
||||
|
|
|
|||
|
|
@ -102,6 +102,44 @@ This container have some options available via environment variables, these opti
|
|||
`false`
|
||||
- <u>Action:</u>
|
||||
Request crond to be run inside the container. Its default configuration will periodically run all scripts from `/etc/periodic/${period}` but custom crontabs can be added to `/var/spool/cron/crontabs/`.
|
||||
- **BACKUP_INTERVAL**:
|
||||
- <u>Possible value:</u>
|
||||
`3h`, `7d`, `3M`
|
||||
- <u>Default:</u>
|
||||
`null`
|
||||
- <u>Action:</u>
|
||||
In combination with `RUN_CROND` set to `true`, enables backup system.\
|
||||
See: [Backup System](#backup-system)
|
||||
- **BACKUP_RETENTION**:
|
||||
- <u>Possible value:</u>
|
||||
`360m`, `7d`, `...m/d`
|
||||
- <u>Default:</u>
|
||||
`7d`
|
||||
- <u>Action:</u>
|
||||
Used by backup system. Backups older than specified in expression are deleted periodically.\
|
||||
See: [Backup System](#backup-system)
|
||||
- **BACKUP_ARG_CONFIG**:
|
||||
- <u>Possible value:</u>
|
||||
`/app/gogs/example/custom/config`
|
||||
- <u>Default:</u>
|
||||
`null`
|
||||
- <u>Action:</u>
|
||||
Used by backup system. If defined, supplies `--config` argument to `gogs backup`.\
|
||||
See: [Backup System](#backup-system)
|
||||
- **BACKUP_ARG_EXCLUDE_REPOS**:
|
||||
- <u>Possible value:</u>
|
||||
`test-repo1`, `test-repo2`
|
||||
- <u>Default:</u>
|
||||
`null`
|
||||
- <u>Action:</u>
|
||||
Used by backup system. If defined, supplies `--exclude-repos` argument to `gogs backup`.\
|
||||
See: [Backup System](#backup-system)
|
||||
|
||||
## Backup System
|
||||
Automated backups with retention policy:
|
||||
|
||||
- `BACKUP_INTERVAL` controls how often the backup job runs and supports interval in hours (h), days (d), and months (M), eg. `3h`, `7d`, `3M`. The lowest possible value is one hour (`1h`).
|
||||
- `BACKUP_RETENTION` supports expressions in minutes (m) and days (d), eg. `360m`, `2d`. The lowest possible value is 60 minutes (`60m`).
|
||||
|
||||
## Upgrade
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,140 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
BACKUP_PATH="/backup"
|
||||
|
||||
# Make sure that required directories exist
|
||||
mkdir -p "${BACKUP_PATH}"
|
||||
mkdir -p "/etc/crontabs"
|
||||
chown git:git /backup
|
||||
chmod 2770 /backup
|
||||
|
||||
# [string] BACKUP_INTERVAL Period expression
|
||||
# [string] BACKUP_RETENTION Period expression
|
||||
if [ -z "${BACKUP_INTERVAL}" ]; then
|
||||
echo "Backup disabled: BACKUP_INTERVAL has not been found" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${BACKUP_RETENTION}" ]; then
|
||||
echo "Backup retention period is not defined, default to 7 days" 1>&2
|
||||
BACKUP_RETENTION='7d'
|
||||
fi
|
||||
|
||||
# Parse BACKUP_INTERVAL environment variable and generate appropriate cron expression. Backup cron task will be run as scheduled.
|
||||
# Expected format: nu (n - number, u - unit) (eg. 3d means 3 days)
|
||||
# Supported units: h - hours, d - days, M - months
|
||||
parse_generate_cron_expression() {
|
||||
CRON_EXPR_MINUTES="*"
|
||||
CRON_EXPR_HOURS="*"
|
||||
CRON_EXPR_DAYS="*"
|
||||
CRON_EXPR_MONTHS="*"
|
||||
|
||||
TIME_INTERVAL=$(echo "${BACKUP_INTERVAL}" | sed -e 's/[hdM]$//')
|
||||
TIME_UNIT=$(echo "${BACKUP_INTERVAL}" | sed -e 's/^[0-9]\+//')
|
||||
|
||||
if [ "${TIME_UNIT}" = "h" ]; then
|
||||
if [ ! "${TIME_INTERVAL}" -le 23 ]; then
|
||||
echo "Parse error: Time unit 'h' (hour) cannot be greater than 23" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CRON_EXPR_MINUTES=0
|
||||
CRON_EXPR_HOURS="*/${TIME_INTERVAL}"
|
||||
elif [ "${TIME_UNIT}" = "d" ]; then
|
||||
if [ ! "${TIME_INTERVAL}" -le 30 ]; then
|
||||
echo "Parse error: Time unit 'd' (day) cannot be greater than 30" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CRON_EXPR_MINUTES=0
|
||||
CRON_EXPR_HOURS=0
|
||||
CRON_EXPR_DAYS="*/${TIME_INTERVAL}"
|
||||
elif [ "${TIME_UNIT}" = "M" ]; then
|
||||
if [ ! "${TIME_INTERVAL}" -le 12 ]; then
|
||||
echo "Parse error: Time unit 'M' (month) cannot be greater than 12" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CRON_EXPR_MINUTES=0
|
||||
CRON_EXPR_HOURS=0
|
||||
CRON_EXPR_DAYS="1"
|
||||
CRON_EXPR_MONTHS="*/${TIME_INTERVAL}"
|
||||
else
|
||||
echo "Parse error: BACKUP_INTERVAL expression is invalid" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "${CRON_EXPR_MINUTES} ${CRON_EXPR_HOURS} ${CRON_EXPR_DAYS} ${CRON_EXPR_MONTHS} *"
|
||||
}
|
||||
|
||||
# Parse BACKUP_RETENTION environment variable and generate appropriate find command expression.
|
||||
# Expected format: nu (n - number, u - unit) (eg. 3d means 3 days)
|
||||
# Supported units: m - minutes, d - days
|
||||
parse_generate_retention_expression() {
|
||||
FIND_TIME_EXPR='mtime'
|
||||
|
||||
TIME_INTERVAL=$(echo "${BACKUP_RETENTION}" | sed -e 's/[mhdM]$//')
|
||||
TIME_UNIT=$(echo "${BACKUP_RETENTION}" | sed -e 's/^[0-9]\+//')
|
||||
|
||||
if [ "${TIME_UNIT}" = "m" ]; then
|
||||
if [ "${TIME_INTERVAL}" -le 59 ]; then
|
||||
echo "Warning: Minimal retention is 60m. Value set to 60m" 1>&2
|
||||
TIME_INTERVAL=60
|
||||
fi
|
||||
|
||||
FIND_TIME_EXPR="mmin"
|
||||
elif [ "${TIME_UNIT}" = "h" ]; then
|
||||
echo "Error: Unsupported expression - Try: eg. 120m for 2 hours." 1>&2
|
||||
exit 1
|
||||
elif [ "${TIME_UNIT}" = "d" ]; then
|
||||
FIND_TIME_EXPR="mtime"
|
||||
elif [ "${TIME_UNIT}" = "M" ]; then
|
||||
echo "Error: Unsupported expression - Try: eg. 60d for 2 months." 1>&2
|
||||
exit 1
|
||||
else
|
||||
echo "Parse error: BACKUP_RETENTION expression is invalid" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "${FIND_TIME_EXPR} +${TIME_INTERVAL:-7}"
|
||||
}
|
||||
|
||||
add_backup_cronjob() {
|
||||
CRONTAB_USER="${1:-git}"
|
||||
CRONTAB_FILE="/etc/crontabs/${CRONTAB_USER}"
|
||||
CRONJOB_EXPRESSION="${2:-}"
|
||||
CRONJOB_EXECUTOR="${3:-}"
|
||||
CRONJOB_EXECUTOR_ARGUMENTS="${4:-}"
|
||||
CRONJOB_TASK="${CRONJOB_EXPRESSION} /bin/sh ${CRONJOB_EXECUTOR} ${CRONJOB_EXECUTOR_ARGUMENTS}"
|
||||
|
||||
if [ -f "${CRONTAB_FILE}" ]; then
|
||||
CRONJOB_EXECUTOR_COUNT=$(grep -c "${CRONJOB_EXECUTOR}" "${CRONTAB_FILE}" || exit 0)
|
||||
if [ "${CRONJOB_EXECUTOR_COUNT}" != "0" ]; then
|
||||
echo "Cron job already exists for ${CRONJOB_EXECUTOR}. Updating existing." 1>&2
|
||||
CRONJOB_TASK=$(echo "{CRONJOB_TASK}" | sed 's/\//\\\//g' )
|
||||
CRONJOB_EXECUTOR=$(echo "{CRONJOB_EXECUTOR}" | sed 's/\//\\\//g' )
|
||||
sed -i "/${CRONJOB_EXECUTOR}/c\\${CRONJOB_TASK}" "${CRONTAB_FILE}"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Finally append new line with cron task expression
|
||||
echo "${CRONJOB_TASK}" >>"${CRONTAB_FILE}"
|
||||
}
|
||||
|
||||
CRONTAB_USER=$(awk -v val="${PUID}" -F ":" '$3==val{print $1}' /etc/passwd)
|
||||
|
||||
# Up to this point, it was desirable that interpreter handles the command errors and halts execution upon any error.
|
||||
# From now, we handle the errors our self.
|
||||
set +e
|
||||
RETENTION_EXPRESSION="$(parse_generate_retention_expression)"
|
||||
|
||||
if [ -z "${RETENTION_EXPRESSION}" ]; then
|
||||
echo "Couldn't generate backup retention expression. Aborting backup setup" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Backup rotator cron will run every 5 minutes
|
||||
add_backup_cronjob "${CRONTAB_USER}" "*/5 * * * *" "/app/gogs/docker/runtime/backup-rotator.sh" "'${BACKUP_PATH}' '${RETENTION_EXPRESSION}'"
|
||||
add_backup_cronjob "${CRONTAB_USER}" "$(parse_generate_cron_expression)" "/app/gogs/docker/runtime/backup-job.sh" "'${BACKUP_PATH}'"
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
#!/usr/bin/env sh
|
||||
|
||||
execute_backup_job() {
|
||||
BACKUP_ARG_PATH="${1:-}"
|
||||
BACKUP_ARG_CONFIG="${BACKUP_ARG_CONFIG:-}"
|
||||
BACKUP_ARG_EXCLUDE_REPOS="${BACKUP_ARG_EXCLUDE_REPOS:-}"
|
||||
cd "/app/gogs" || exit 1
|
||||
|
||||
BACKUP_ARGS="--target=${BACKUP_ARG_PATH}"
|
||||
|
||||
if [ -n "${BACKUP_ARG_CONFIG}" ]; then
|
||||
BACKUP_ARGS="${BACKUP_ARGS} --config=${BACKUP_ARG_CONFIG}"
|
||||
fi
|
||||
|
||||
if [ -n "${BACKUP_ARG_EXCLUDE_REPOS}" ]; then
|
||||
BACKUP_ARGS="${BACKUP_ARGS} --exclude-repos=${BACKUP_ARG_EXCLUDE_REPOS}"
|
||||
fi
|
||||
|
||||
./gogs backup "${BACKUP_ARGS}" || echo "Error: Backup job returned non-successful code." && exit 1
|
||||
}
|
||||
|
||||
main() {
|
||||
BACKUP_PATH="${1:-}"
|
||||
|
||||
if [ -z "${BACKUP_PATH}" ]; then
|
||||
echo "Required argument missing BACKUP_PATH" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
execute_backup_job "${BACKUP_PATH}"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
#!/usr/bin/env sh
|
||||
|
||||
# This is very simple, yet effective backup rotation script.
|
||||
# Using find command, all files that are older than BACKUP_RETENTION_DAYS are accumulated and deleted using rm.
|
||||
main() {
|
||||
BACKUP_PATH="${1:-}"
|
||||
FIND_EXPRESSION="${2:-mtime +7}"
|
||||
|
||||
if [ -z "${BACKUP_PATH}" ]; then
|
||||
echo "Error: Required argument missing BACKUP_PATH" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$(realpath "${BACKUP_PATH}")" = "/" ]; then
|
||||
echo "Error: Dangerous BACKUP_PATH: /" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "${BACKUP_PATH}" ]; then
|
||||
echo "Error: BACKUP_PATH does't exist or is not a directory" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
find "${BACKUP_PATH}/" -type f -name "gogs-backup-*.zip" -${FIND_EXPRESSION} -print -exec rm "{}" +
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
|
@ -64,6 +64,7 @@ CROND=$(echo "$RUN_CROND" | tr '[:upper:]' '[:lower:]')
|
|||
if [ "$CROND" = "true" -o "$CROND" = "1" ]; then
|
||||
echo "init:crond | Cron Daemon (crond) will be run as requested by s6" 1>&2
|
||||
rm -f /app/gogs/docker/s6/crond/down
|
||||
/bin/sh /app/gogs/docker/runtime/backup-init.sh "${PUID}"
|
||||
else
|
||||
# Tell s6 not to run the crond service
|
||||
touch /app/gogs/docker/s6/crond/down
|
||||
|
|
|
|||
Loading…
Reference in New Issue