docker-adsb-ultrafeeder/rootfs/etc/s6-overlay/scripts/mlat-client

#!/command/with-contenv bash
# shellcheck shell=bash disable=SC1091,SC2015,SC2016,SC2001

#---------------------------------------------------------------------------------------------
# Copyright (C) 2023-2024, Ramon F. Kolb (kx1t) and contributors
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with this program.
# If not, see <https://www.gnu.org/licenses/>.
#---------------------------------------------------------------------------------------------

trap 'pkill -P "$$" || true; s6wrap --quiet --timestamps --prepend=mlat-client --args echo "service stopping"; exit 0' SIGTERM SIGINT SIGHUP SIGQUIT

source /scripts/common
source /scripts/interpret_ultrafeeder_config

s6wrap=(s6wrap --quiet --timestamps --prepend="$SCRIPT_NAME")

# run with slightly lower priority
renice 5 $$ || true

"${s6wrap[@]}" --args echo "Started as an s6 service"

MLAT_CMD="/usr/bin/mlat-client"

LOGLEVEL="${LOGLEVEL,,}"
LOGLEVEL="${LOGLEVEL:-verbose}"
chk_disabled "$LOGLEVEL" && LOGLEVEL="none" || true
if ! [[ "$LOGLEVEL" =~ ^(verbose|error|none)$ ]]; then 
    [[ -n "$LOGLEVEL" ]] && "${s6wrap[@]}" --args echo "[WARNING] LOGLEVEL set to an unknown value. Defaulting to \"verbose\"" || true
    LOGLEVEL="verbose"
fi

RESTARTTIMER=15
declare -A pid_array

if [[ -z "${MLAT_CONFIG}" ]]
then
    "${s6wrap[@]}" --args echo "Warning: MLAT_CONFIG not defined - MLAT will be disabled."
    stop_service
fi

if [[ -z "${MLAT_USER}" ]] && [[ -z "${UUID}" ]]
then
    "${s6wrap[@]}" --args echo "ERROR: either UUID or MLAT_USER must be defined - MLAT will be disabled."
    stop_service
fi

mkdir -p /run/mlat-client

function check_gpsd() {
    if (( GPSD == 0 )) || ! [[ -f /run/readsb/gpsd.json ]]; then
        return 1
    fi
    if new_lat="$(jq -r .lat /run/readsb/gpsd.json)" \
        && [[ "$new_lat" != "null" ]] \
        && new_lon="$(jq -r .lon /run/readsb/gpsd.json)" \
        && [[ "$new_lon" != "null" ]] \
        && new_alt="$(jq -r .altMSL /run/readsb/gpsd.json)" \
        && [[ "$new_alt" != "null" ]] \

    then
        # yay, vars are set and not null
        return 0
    else
        new_lat=""
        new_lon=""
        new_alt=""
        return 1
    fi
}

GPSD=0
if grep -qs "gpsd" <<< "$ULTRAFEEDER_CONFIG" || grep -qs "gpsd" <<< "$ULTRAFEEDER_NET_CONNECTOR"; then
    GPSD=1
    LOCATION_PERSIST=/var/globe_history/gpsd_last_location
    if [[ -f "$LOCATION_PERSIST" ]]; then
        read new_lat new_lon new_alt < "$LOCATION_PERSIST"
    fi
    # initialize "old" location for gpsd movement detection
    # use zero island as starting point if location persist does not exit
    old_lat=${new_lat:-0}
    old_lon=${new_lon:-0}

    # wait for gpsd to continue with startup
    "${s6wrap[@]}" --args echo "GPSD configured, waiting for gpsd to provide location data"
    while ! check_gpsd; do
        sleep "${GPSD_CHECK_INTERVAL:-30}" & wait $!
    done
    "${s6wrap[@]}" --args echo "GPSD has provided location data"

    GPSD_MIN_DISTANCE="${GPSD_MIN_DISTANCE:-20}"
    # enforce gpsd min distance is no larger than 40m
    if (( GPSD_MIN_DISTANCE > 40 )); then
        GPSD_MIN_DISTANCE=40
    fi

    GPSD_CHECK_INTERVAL="${GPSD_CHECK_INTERVAL:-30}"
    if (( GPSD_CHECK_INTERVAL < 5 )); then
        GPSD_CHECK_INTERVAL=5
    fi

    # in seconds
    no_movement_required=${GPSD_MLAT_WAIT:-90}
    # enforce it to be longer than the checking interval for implementation reasons
    if (( no_movement_required < GPSD_CHECK_INTERVAL )); then
        no_movement_required="${GPSD_CHECK_INTERVAL}"
    fi
    # enforce 90 second minimum
    if (( no_movement_required < 90 )); then
        no_movement_required=90
    fi
    # set no_movement to a number higher than the required time of no movement
    # this way on startup there is no bogus message printed about starting mlat-clients with a new location
    no_movement=$(( 2 * no_movement_required ))
else
    if [[ -z "$LAT$READSB_LAT" ]]; then
        "${s6wrap[@]}" --args echo "ERROR: READSB_LAT or LAT must be defined or GPSD must be enabled - MLAT will be disabled."
        exec sleep infinity
    fi
    if [[ -z "$LONG$READSB_LON" ]]; then
        "${s6wrap[@]}" --args echo "ERROR: READSB_LON or LONG must be defined or GPSD must be enabled - MLAT will be disabled."
        exec sleep infinity
    fi
    if [[ -z "$ALT$READSB_ALT" ]]; then
        "${s6wrap[@]}" --args echo "ERROR: READSB_ALT or ALT must be defined or GPSD must be enabled - MLAT will be disabled."
        exec sleep infinity
    fi
fi

# MLAT_CONFIG has the following format:
# MLAT_CONFIG=mlatserver_1,mlatserver_port_1,x1,y1,z1;mlatserver_2,mlatserver_port_2,x2,y2,z2 etc
# where x1,y1,z1; x22,y2,z2; etc are optional and are interpreted as follows:
# if it's a number, we'll assume it's a port number for return messages
# if it starts with 'uuid=', it's a UUID number for that instance
# anything else will be appended as extra parameter(s) to the mlat-client command line

# parse MLAT_CONFIG string into mlat_configs array
# Strip any extraneous spaces:
MLAT_CONFIG="${MLAT_CONFIG#"${MLAT_CONFIG%%[![:space:]]*}"}"   # strip leading space
MLAT_CONFIG="${MLAT_CONFIG//; /;}"
# remove any newlines:
MLAT_CONFIG="${MLAT_CONFIG//$'\n'/}"
readarray -td ";" mlat_configs < <(printf '%s' "${MLAT_CONFIG}")


# wait until readsb is established...
if ! pgrep readsb >/dev/null; then
    if [[ -z "${LOGLEVEL}" ]] || [[ "${LOGLEVEL,,}" == "verbose" ]]; then
        "${s6wrap[@]}" --args echo "Delaying start of MLAT client(s) until container is established..."
    fi
    while ! pgrep readsb >/dev/null
        do
        sleep 2 & wait $!
    done
    # wait 2 seconds after readsb process exists
    sleep 2 & wait $!
fi


# Now loop through the MLAT_CONFIG items and start up an Mlat_client for each of them:
for instance in "${mlat_configs[@]}"
do
    [[ -z "${instance}" ]] && continue || true
    # put individual params into the $params array:
    readarray -td "," params < <(printf '%s' "${instance}")

    # Check if the params array has values for the mandatory elements:
    if [[ -z "${params[0]}" ]] || [[ -z "${params[1]}" ]]
    then
        "${s6wrap[@]}" --args echo "ERROR -- MLAT_CONFIG is malformed: \"${instance}\". Stopping MLAT execution."
         # shellcheck disable=SC2046
        kill $(ps -s $$ -o pid=)
        exec sleep infinity
    fi

    # Now process the the arguments
    # The order doesn't matter, we'll do pattern matching:
    # If the argument is a number, then it must be the beast_results port
    # If the argument starts with "uuid=", then it must be a UUID, etc.
    # If the argument isn't any of the above, then it's an "extra argument"

    unset name_arg uuid_arg lat_arg lon_arg alt_arg input_connect_arg return_port_arg extra_args

    for ((i=2; i<${#params[*]}; i++))
    do
        if [[ -n "${params[i]}" ]] && [[ "${params[i]}" =~ ^[0-9]+$ ]]; then
            # It's a number so it must be the return port
            return_port_arg="${params[i]}"
        elif header="${params[i]:0:5}" && [[ "${header,,}" == "uuid=" ]]; then
            # It's a UUID
            uuid_arg="${params[i]#*=}"
            elif header="${params[i]:0:5}" && [[ "${header,,}" == "name=" ]]; then
            # It's a MLAT_NAME
            name_arg="${params[i]#*=}"
        elif header="${params[i]:0:4}" && [[ "${header,,}" == "lat=" ]]; then
            # It's a latitude
            lat_arg="${params[i]#*=}"
        elif header="${params[i]:0:4}" && [[ "${header,,}" == "lon=" ]]; then
            # It's a longitude
            lon_arg="${params[i]#*=}"
        elif header="${params[i]:0:4}" && [[ "${header,,}" == "alt=" ]]; then
            # It's a latitude
            alt_arg="${params[i]#*=}"
        elif header="${params[i]:0:14}" && [[ "${header,,}" == "input_connect=" ]]; then
            #It's the input_connect parameter value
            input_connect_arg="${params[i]#*=}"
        else
            # It's an Extra Args string
            extra_args="$extra_args ${params[i]}"
        fi
    done

    # ------------------------------------------------
    # Build the MLAT parameter string:
    MLAT_PARAM=(--input-type "${MLAT_INPUT_TYPE:-auto}")
    MLAT_PARAM+=(--server "${params[0]}:${params[1]}")

    servername=${params[0]}

    # add return port if defined
    if [[ -n "${return_port_arg}" ]] && ! chk_disabled "${return_port_arg}"; then
        MLAT_PARAM+=("--results beast,listen,${return_port_arg}")
    fi
    # send data to MLATHUB unless disabled
    if ! chk_disabled "${return_port_arg}" && ! chk_enabled "${MLATHUB_DISABLE}"; then
        MLAT_PARAM+=("--results beast,connect,localhost:${MLATHUB_BEAST_IN_PORT:-31004}")
    fi

    # add input-connect to the param array:
    MLAT_PARAM+=(--input-connect "${input_connect_arg:-localhost:30005}")

    if [[ -n "${name_arg}" ]] || [[ -n "${MLAT_USER}" ]]; then
        MLAT_PARAM+=(--user \""${name_arg:-${MLAT_USER}}"\")
    else
        rnd="${RANDOM}"
        "${s6wrap[@]}" --args echo "WARNING: MLAT_USER is not set - using random number \"${rnd}\" as MLAT_USER"
        MLAT_PARAM+=(--user "${rnd}")
    fi

    # add LAT/LON/ALT to instance:

    if [[ -n "${new_lat:-$lat_arg}" ]]; then
        MLAT_PARAM+=(--lat "${new_lat:-$lat_arg}")
    elif [[ -n "${LAT}" ]]; then
        MLAT_PARAM+=(--lat "${LAT}")
    elif [[ -n "${READSB_LAT}" ]]; then
        MLAT_PARAM+=(--lat "${READSB_LAT}")
    fi

    if [[ -n "${new_lon:-$lon_arg}" ]]; then
        MLAT_PARAM+=(--lon "${new_lon:-$lon_arg}")
    elif [[ -n "${LONG}" ]]; then
        MLAT_PARAM+=(--lon "${LONG}")
    elif [[ -n "${READSB_LON}" ]]; then
        MLAT_PARAM+=(--lon "${READSB_LON}")
    fi

    if [[ -n "${new_alt:-$alt_arg}" ]]; then
        MLAT_PARAM+=(--alt "${new_alt:-$alt_arg}")
    elif [[ -n "${ALT}" ]]; then
        MLAT_PARAM+=(--alt "${ALT}")
    elif [[ -n "${READSB_ALT}" ]]; then
        MLAT_PARAM+=(--alt "${READSB_ALT}")
    fi

    # Add UUID to instance
    if [[ -n "$uuid_arg" ]]; then
        MLAT_PARAM+=("--uuid ${uuid_arg}")
    elif [[ -n "${UUID}" ]]; then
        MLAT_PARAM+=("--uuid ${UUID}")
    else
        "${s6wrap[@]}" --args echo "WARNING: UUID is not defined, proceeding without UUID"
    fi

    MLAT_PARAM+=(--stats-json "/run/mlat-client/${params[0]}:${params[1]}.json")

    # Now add the extra_args, if any:
    [[ -n "${extra_args}" ]] && MLAT_PARAM+=("${extra_args}") || true

    # ------------------------------------------------
    # Create the command exec string:
    # shellcheck disable=SC2048,SC2086
    execstring="$(echo ${MLAT_CMD} ${MLAT_PARAM[*]} | xargs)"

    if (( GPSD == 1 )); then
        # when GPSD is active, just write the pid array, mlat-client will be started by the gpsd checking logic later
        # use a long random PID so that it's detected as "not running"
        pid_array["${RANDOM}${RANDOM}${RANDOM}"]="${MLAT_PARAM[*]}"
        continue
    fi

    # stagger by 15 second so they don't all start at the same time
    sleep "${MLAT_STARTUP_STAGGER:-15}" & wait $!

    # ------------------------------------------------
    # run this Mlat_client instance in the background:

    #shellcheck disable=SC2069,SC2086
    if [[ "${LOGLEVEL}" == "verbose" ]]; then
        "${s6wrap[@]}" --prepend="$(basename "$0")][${servername}" --args echo "starting: ${execstring}"
        "${s6wrap[@]}" --prepend="$(basename "$0")][${servername}" --args ${execstring} &
    elif [[ "${LOGLEVEL}" == "error" ]]; then
        "${s6wrap[@]}" --ignore=stdout --prepend="$(basename "$0")][${servername}" --args ${execstring} &
    elif [[ "${LOGLEVEL}" == "none" ]]; then
        "${s6wrap[@]}" --ignore=stderr --ignore=stdout --prepend="$(basename "$0")][${servername}" --args ${execstring} &
    fi

    # pid_array is indexed by the PID of each mlat_client and contains the MLAT_PARAMs for that instance
    # This is done so we can monitor them and restart them if needed
    pid_array[$!]="${MLAT_PARAM[*]}"
done

sleep 5 & wait $!

# Now iterate over all MLAT-client instances and check if they are still running:
while true
do
    if (( GPSD == 1 )); then
        if ! check_gpsd; then
            # don't do mlat client restarts if GPSD is configured but not providing a position
            sleep "${GPSD_CHECK_INTERVAL}" & wait $!
            continue
        fi

        distance="$(distance "$old_lat" "$old_lon" "$new_lat" "$new_lon")"
        if ! [[ -f "$LOCATION_PERSIST" ]]; then
            echo "$new_lat" "$new_lon" "$new_alt" > "$LOCATION_PERSIST"
        fi
        if (( ${distance%%.*} > ${GPSD_MIN_DISTANCE:-20} )); then

            msg="Receiver moved ${distance%%.*} meters"

            # kill the mlat-client instances so they get restarted with the new GPS coords
            if pkill -f "/usr/bin/python3 /usr/bin/mlat-client" >/dev/null 2>&1; then
                msg+=" - Stopping all mlat-clients"
            fi

            "${s6wrap[@]}" --args echo "${msg}"

            old_lat="$new_lat"
            old_lon="$new_lon"

            # new location means the receiver has moved, sleep a bit and then check again
            no_movement=0
            sleep "${GPSD_CHECK_INTERVAL:-30}" & wait $!
            # as the recevier has moved, mlat-clients are not restarted until there has been no movement for some time
            # thus we continue skipping the mlat-client restart section of the loop
            continue
        else
            # no movement during the checking interval, allow mlat-clients to be restarted
            (( no_movement += ${GPSD_CHECK_INTERVAL:-30} )) || true
            if (( no_movement < no_movement_required )); then
                msg="Receiver moved less than ${GPSD_MIN_DISTANCE} meters, $(( no_movement_required - no_movement )) seconds remaining before starting mlat-clients"
                "${s6wrap[@]}" --args echo "${msg}"
                sleep "${GPSD_CHECK_INTERVAL:-30}" & wait $!
                continue
            elif (( no_movement / ${GPSD_CHECK_INTERVAL:-30} == no_movement_required / ${GPSD_CHECK_INTERVAL:-30} )); then
                "${s6wrap[@]}" --args echo "Receiver stationary - starting all mlat-clients with new location"
                echo "$new_lat" "$new_lon" "$new_alt" > "$LOCATION_PERSIST"
            fi
        fi
    fi

    for mlat_pid in "${!pid_array[@]}"
    do
        if ! kill -0 "${mlat_pid}" >/dev/null 2>&1
        then
            # it exited - let's restart:
            if [[ ! -f /run/readsb/gpsd.json ]] || [[ "$(jq -r .lat /run/readsb/gpsd.json)" == "null" ]]; then
                # only sleep for a bit if the restarts aren't caused by GPS movement:
                sleep "${RESTARTTIMER}" & wait $!
            fi
            servername="$(awk '{print $4}' <<< "${pid_array[$mlat_pid]}")"
            servername="${servername%%:*}"

            [[ "${LOGLEVEL,,}" != "none" ]] && "${s6wrap[@]}" --prepend="$(basename "$0")[${servername}" --args echo "MLAT_Client ${servername} exited. Attempting to restart." || true
            # shellcheck disable=SC2086
            execstring="$(echo ${MLAT_CMD} ${pid_array[$mlat_pid]} | xargs)"

            # If GPSD is active, then replace the lat/lon/alt params with the ones from GPSD
            if (( GPSD == 1 )); then
                execstring="$(sed "s/^\(.*\s\+--lat\s\+\)[0-9.-]\+\(.*\)$/\1${new_lat}\2/g" <<< "$execstring")"
                execstring="$(sed "s/^\(.*\s\+--lon\s\+\)[0-9.-]\+\(.*\)$/\1${new_lon}\2/g" <<< "$execstring")"
                execstring="$(sed "s/^\(.*\s\+--alt\s\+\)[mft0-9.-]\+\(.*\)$/\1${new_alt}m\2/g" <<< "$execstring")"
            fi

            #shellcheck disable=SC2069,SC2086
            if [[ "${LOGLEVEL}" == "verbose" ]]; then
                "${s6wrap[@]}" --prepend="$(basename "$0")][${servername}" --args echo "Restarting: ${execstring}"
                "${s6wrap[@]}" --prepend="$(basename "$0")][${servername}" --args ${execstring} &
            elif [[ "${LOGLEVEL}" == "error" ]]; then
                "${s6wrap[@]}" --ignore=stdout --prepend="$(basename "$0")][${servername}" --args ${execstring} &
            elif [[ "${LOGLEVEL}" == "none" ]]; then
                "${s6wrap[@]}" --ignore=stderr --ignore=stdout --prepend="$(basename "$0")][${servername}" --args ${execstring} &
            fi

            pid_array[$!]="${pid_array[${mlat_pid}]}"
            unset "pid_array[${mlat_pid}]"
        fi
    done

    sleep "${GPSD_CHECK_INTERVAL:-30}" & wait $!
done