From be5df3e3ded1c7a8f94838685e2388a6db8ec1c2 Mon Sep 17 00:00:00 2001 From: kx1t <15090643+kx1t@users.noreply.github.com> Date: Tue, 23 Jul 2024 15:47:06 -0400 Subject: [PATCH] add purging option for `globe_history` with `MAX_GLOBE_HISTORY` parameter (#115) * add AussieADSB registration script * update readme * make independent of container run * update * update readme * minor update * fix typo * Update aussieadsb.sh * Update aussieadsb.sh * Update aussieadsb.sh * Add netcat-traditional and net-tools to install check * Update README.md * Update aussieadsb.sh * Update aussieadsb.sh * Dockerfile building optimization * typo * typo * add cleanup_globe_history * minor update * better logging --- Dockerfile | 34 ++++++++-------- README.md | 1 + .../dependencies.d/startup | 0 .../s6-rc.d/cleanup_globe_history/run | 2 + .../s6-rc.d/cleanup_globe_history/type | 1 + .../user/contents.d/cleanup_globe_history | 0 .../s6-overlay/scripts/cleanup_globe_history | 40 +++++++++++++++++++ rootfs/scripts/interpret_ultrafeeder_config | 10 ++++- 8 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 rootfs/etc/s6-overlay/s6-rc.d/cleanup_globe_history/dependencies.d/startup create mode 100755 rootfs/etc/s6-overlay/s6-rc.d/cleanup_globe_history/run create mode 100644 rootfs/etc/s6-overlay/s6-rc.d/cleanup_globe_history/type create mode 100755 rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/cleanup_globe_history create mode 100755 rootfs/etc/s6-overlay/scripts/cleanup_globe_history diff --git a/Dockerfile b/Dockerfile index e2d8a1a..285e7b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,12 @@ -FROM ghcr.io/sdr-enthusiasts/docker-baseimage:mlatclient as mlatclient +FROM ghcr.io/sdr-enthusiasts/docker-baseimage:mlatclient as buildimage + +SHELL ["/bin/bash", "-x", "-o", "pipefail", "-c"] +RUN \ + --mount=type=bind,source=./,target=/app/ \ + apt-get update -q -y && \ + apt-get install -o Dpkg::Options::="--force-confnew" -y --no-install-recommends -q \ + build-essential && \ + gcc -static /app/downloads/distance-in-meters.c -o /distance -lm -O2 FROM ghcr.io/sdr-enthusiasts/docker-tar1090:latest @@ -8,19 +16,17 @@ ENV URL_MLAT_CLIENT_REPO="https://github.com/wiedehopf/mlat-client.git" \ PRIVATE_MLAT="false" \ MLAT_INPUT_TYPE="auto" +ARG VERSION_REPO="sdr-enthusiasts/docker-adsb-ultrafeeder" \ + VERSION_BRANCH="##BRANCH##" + SHELL ["/bin/bash", "-x", "-o", "pipefail", "-c"] RUN \ - --mount=type=bind,source=./,target=/app/ \ - --mount=type=bind,from=mlatclient,source=/,target=/mlatclient/ \ + --mount=type=bind,from=buildimage,source=/,target=/buildimage/ \ TEMP_PACKAGES=() && \ KEPT_PACKAGES=() && \ # Needed to run the mlat_client: KEPT_PACKAGES+=(python3-minimal) && \ KEPT_PACKAGES+=(python3-pkg-resources) && \ - # needed to compile distance - TEMP_PACKAGES+=(build-essential) && \ - # needed for container version - TEMP_PACKAGES+=(git) && \ # # packages needed for debugging - these can stay out in production builds: #KEPT_PACKAGES+=(procps nano aptitude psmisc) && \ @@ -30,17 +36,13 @@ RUN \ "${KEPT_PACKAGES[@]}" \ "${TEMP_PACKAGES[@]}" && \ # Get mlat-client - tar zxf /mlatclient/mlatclient.tgz -C / && \ + tar zxf /buildimage/mlatclient.tgz -C / && \ ln -s /usr/local/bin/mlat-client /usr/bin/mlat-client && \ - # Compile distance binary - gcc -static /app/downloads/distance-in-meters.c -o /usr/local/bin/distance -lm -O2 && \ + # Get distance binary + cp -f /buildimage/distance /usr/local/bin/distance && \ # Add Container Version - branch="##BRANCH##" && \ - [[ "${branch:0:1}" == "#" ]] && branch="main" || true && \ - git clone --depth=1 -b "$branch" https://github.com/sdr-enthusiasts/docker-adsb-ultrafeeder.git /tmp/clone && \ - pushd /tmp/clone && \ - echo "$(TZ=UTC date +%Y%m%d-%H%M%S)_$(git rev-parse --short HEAD)_$(git branch --show-current)" > /.CONTAINER_VERSION && \ - popd && \ + [[ "${VERSION_BRANCH:0:1}" == "#" ]] && VERSION_BRANCH="main" || true && \ + echo "$(TZ=UTC date +%Y%m%d-%H%M%S)_$(curl -ssL https://api.github.com/repos/$VERSION_REPO/commits/$VERSION_BRANCH | awk '{if ($1=="\"sha\":") {print substr($2,2,7); exit}}')_$VERSION_BRANCH" > /.CONTAINER_VERSION && \ # Clean up and install POST_PACKAGES: apt-get remove -q -y "${TEMP_PACKAGES[@]}" && \ # apt-get install -o Dpkg::Options::="--force-confnew" -y --no-install-recommends -q \ diff --git a/README.md b/README.md index 583ec76..bacd7ba 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,7 @@ The following parameters must be set (mandatory) for the container to function: | `S6_SERVICES_GRACETIME` | Optional, set to 30000 when saving traces / globe_history | | `3000` | | `LOGLEVEL` | `verbose` (all messages), `error` (errors only), `none` (minimal) | | `verbose` | | `MLAT_STARTUP_STAGGER` | mlat-client startup staggering for tidy logs (reduce for quicker startup) | | `15` | +| `MAX_GLOBE_HISTORY` | Maximum number of days that `globe_history` data (used to produce heatmaps and ptracks) is retained. Note - this parameter doesn't affect the data used to produce `graphs1090` statistics | | Unset | `READSB_EXTRA_ARGS` just passes arguments to the commandline, you can check this file for more options for wiedehopf's readsb fork: diff --git a/rootfs/etc/s6-overlay/s6-rc.d/cleanup_globe_history/dependencies.d/startup b/rootfs/etc/s6-overlay/s6-rc.d/cleanup_globe_history/dependencies.d/startup new file mode 100644 index 0000000..e69de29 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/cleanup_globe_history/run b/rootfs/etc/s6-overlay/s6-rc.d/cleanup_globe_history/run new file mode 100755 index 0000000..ea60751 --- /dev/null +++ b/rootfs/etc/s6-overlay/s6-rc.d/cleanup_globe_history/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec /etc/s6-overlay/scripts/cleanup_globe_history diff --git a/rootfs/etc/s6-overlay/s6-rc.d/cleanup_globe_history/type b/rootfs/etc/s6-overlay/s6-rc.d/cleanup_globe_history/type new file mode 100644 index 0000000..5883cff --- /dev/null +++ b/rootfs/etc/s6-overlay/s6-rc.d/cleanup_globe_history/type @@ -0,0 +1 @@ +longrun diff --git a/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/cleanup_globe_history b/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/cleanup_globe_history new file mode 100755 index 0000000..e69de29 diff --git a/rootfs/etc/s6-overlay/scripts/cleanup_globe_history b/rootfs/etc/s6-overlay/scripts/cleanup_globe_history new file mode 100755 index 0000000..bfd5031 --- /dev/null +++ b/rootfs/etc/s6-overlay/scripts/cleanup_globe_history @@ -0,0 +1,40 @@ +#!/command/with-contenv bash +# shellcheck shell=bash disable=SC2015,SC2016,SC1091,SC2001,SC2154 + +#--------------------------------------------------------------------------------------------- +# Script to remove Globe_History files older than xxx days - this to ensure that the disk +# doesn't fill up with (unwanted) history files) + +# Copyright (C) 2023-2024, Ramon F. Kolb (kx1t) and contributors +# Core script copyright and provided by Matthias Wirth (wiedehopf), used with permission +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with this program. +# If not, see . +#--------------------------------------------------------------------------------------------- + +source /scripts/common + +if [[ -z "$MAX_GLOBE_HISTORY" ]]; then + "${s6wrap[@]}" echo "MAX_GLOBE_HISTORY not set - we will not expire any globe_history files" + exec sleep infinity +fi + +cutoffepoch="$(date -d"-${MAX_GLOBE_HISTORY} days" +%s)" + +"${s6wrap[@]}" echo "Purging globe_history older than $MAX_GLOBE_HISTORY days (before $(date -d"-${MAX_GLOBE_HISTORY} days" +%d-%b-%Y))" +for dir in $(find /var/globe_history -maxdepth 3 -mindepth 3 | grep -o -E -e '[0-9]{4}/[0-9]{2}/[0-9]{2}$'); do + if (( $(date -d "$dir" +%s) < cutoffepoch )); then + rm -rf "/var/globe_history/$dir" + fi +done +"${s6wrap[@]}" echo "Done - next purge will be in 24 hours" +exec sleep 24h diff --git a/rootfs/scripts/interpret_ultrafeeder_config b/rootfs/scripts/interpret_ultrafeeder_config index ce70123..1ea517a 100755 --- a/rootfs/scripts/interpret_ultrafeeder_config +++ b/rootfs/scripts/interpret_ultrafeeder_config @@ -1,6 +1,6 @@ #!/usr/bin/with-contenv bash -# shellcheck shell=bash disable=SC2015 +# shellcheck shell=bash disable=SC2015,SC2154 # # This scripts should be sourced by the /etc/services.d/xxx/run modules for # readsb, mlat-client, and mlathub. It interprets the ULTRAFEEDER_CONFIG / ULTRAFEEDER_NET_CONNECTOR parameter @@ -97,6 +97,14 @@ do done # Now add $readsb_str to the READSB_CONF_ARR() as --net-connector argument: READSB_CONF_ARR+=("--net-connector=${readsb_str}") + + if [[ ${param[1],,} == "in.adsb.lol" ]]; then + myadsblol="$(curl -sLD - https://my.adsb.lol -o /dev/null -w '%{url_effective}' 2>&1 | sed '/location: /h;g;$!d;s/^.* \(.*\)$/\1/')" + if [[ -n "$myadsblol" ]] && [[ ! $myadsblol =~ sorry-but-i-could-not-find-your-receiver ]]; then + "${s6wrap[@]}" echo "ADSB.LOL station map available at $myadsblol" + fi + fi + ;; mlat)