diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 83c7b6fe..62c49cbd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,7 +61,7 @@ jobs: - name: Start run: | - docker compose up -d + docker compose up -d --wait --wait-timeout 180 working-directory: test env: LIBRENMS_IMAGE: ${{ env.BUILD_TAG }} diff --git a/Dockerfile b/Dockerfile index f747f622..871dff5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -149,3 +149,6 @@ EXPOSE 8000 514 514/udp 162 162/udp VOLUME [ "/data" ] ENTRYPOINT [ "/init" ] + +HEALTHCHECK --start-period=1m --timeout=10s \ + CMD /usr/local/bin/healthcheck diff --git a/rootfs/etc/s6-overlay/scripts/config-dispatcher b/rootfs/etc/s6-overlay/scripts/config-dispatcher index f9525c85..8e345c03 100755 --- a/rootfs/etc/s6-overlay/scripts/config-dispatcher +++ b/rootfs/etc/s6-overlay/scripts/config-dispatcher @@ -54,6 +54,12 @@ echo ">>" echo ">> Sidecar dispatcher container detected" echo ">>" +cat >"${LIBRENMS_PATH}/config.d/dispatcher-health.php" <<'EOL' +&2 "ERROR: Either DB_PASSWORD or DB_PASSWORD_FILE must be defined" diff --git a/rootfs/usr/local/bin/healthcheck b/rootfs/usr/local/bin/healthcheck new file mode 100755 index 00000000..d8646d21 --- /dev/null +++ b/rootfs/usr/local/bin/healthcheck @@ -0,0 +1,31 @@ +#!/usr/bin/env sh +set -eu + +services="" +check_librenms=0 +if [ "${SIDECAR_DISPATCHER:-0}" = "1" ]; then + services="$services dispatcher" + check_librenms=1 +fi +if [ "${SIDECAR_SYSLOGNG:-0}" = "1" ]; then + services="$services syslogng" +fi +if [ "${SIDECAR_SNMPTRAPD:-0}" = "1" ]; then + services="$services snmptrapd" +fi + +if [ -z "$services" ]; then + services="nginx php-fpm snmpd cron" + check_librenms=1 +fi + +for service in $services; do + if [ "$(s6-svstat -o up,wantedup "/run/service/$service")" != "true true" ]; then + echo >&2 "Service $service is not running" + exit 1 + fi +done + +if [ "$check_librenms" = "1" ]; then + exec lnms health:check +fi diff --git a/test/services.sh b/test/services.sh index e9fc3831..0be297d3 100644 --- a/test/services.sh +++ b/test/services.sh @@ -34,3 +34,29 @@ for service in $services; do ;; esac done + +healthcheck +if [ "${SIDECAR_DISPATCHER:-0}" = "1" ]; then + # Freeze the dispatcher so it cannot refresh the stale heartbeat during the check. + s6-svc -p /run/service/dispatcher + touch -d '@1' /opt/librenms/storage/dispatcher-health + if healthcheck; then + s6-svc -c /run/service/dispatcher + echo >&2 "Healthcheck accepted a stale dispatcher heartbeat" + exit 1 + fi + s6-svc -c /run/service/dispatcher +fi + +for service in $services; do + s6-svc -d "/run/service/$service" + s6-svwait -d -t 120000 "/run/service/$service" + if healthcheck; then + s6-svc -u "/run/service/$service" + echo >&2 "Healthcheck accepted stopped service $service" + exit 1 + fi + s6-svc -u "/run/service/$service" + s6-svwait -u -t 10000 "/run/service/$service" +done +timeout 30 sh -c 'until healthcheck; do sleep 1; done'