RSS tracker added. Requires xmlstarlet.

This commit is contained in:
Storm Dragon
2026-07-13 15:19:28 -04:00
parent 5e749af65b
commit 13d44cf55e
5 changed files with 610 additions and 0 deletions
+2
View File
@@ -2,6 +2,8 @@ response/error.txt
response/exit.txt
triggers/greet/greetings.txt
triggers/keywords/keywords.cfg
services/rss/rss.conf
services/rss/rss.seen
log.txt
.*
bot.cfg
+10
View File
@@ -25,3 +25,13 @@ Throw "modules" into /modules/module-name/module-name.sh
and they will be loaded up during runtime.
Call modules with botname: test arg1 arg2 ....
## Services
Background services live under /services/service-name/service-name.sh and are
started by the bot when it connects.
The RSS service is configured in services/rss/rss.conf, which is created from
services/rss/rss.conf.example on first run. It requires curl and xmlstarlet,
marks existing feed items read on first startup, then announces new items to
all configured channels.
+27
View File
@@ -25,6 +25,7 @@ check_config_files() {
"response/exit.txt:response/exit.txt.example"
"triggers/greet/greetings.txt:triggers/greet/greetings.txt.example"
"triggers/keywords/keywords.cfg:triggers/keywords/keywords.cfg.example"
"services/rss/rss.conf:services/rss/rss.conf.example"
)
for entry in "${configFiles[@]}"; do
@@ -201,6 +202,29 @@ start_ping_monitor() {
pingMonitorPid=$!
}
# Function to stop RSS service process
stop_rss_service() {
if [[ -n "$rssServicePid" ]] && kill -0 "$rssServicePid" 2>/dev/null; then
kill "$rssServicePid" 2>/dev/null
wait "$rssServicePid" 2>/dev/null
fi
}
# Start background RSS service process
start_rss_service() {
if [[ ! -x "./services/rss/rss.sh" ]]; then
return
fi
if [[ ${#channels[@]} -eq 0 ]]; then
return
fi
stop_rss_service
"./services/rss/rss.sh" "$input" "$log" "$dateFormat" "${channels[@]}" &
rssServicePid=$!
}
# Function to trim log file to prevent unbounded growth
# Keeps log between 800-1000 lines by trimming oldest entries when limit reached
trim_log() {
@@ -225,6 +249,7 @@ trim_log() {
# Trap exiting from the program to remove the temporary input file and stop ping monitor
cleanup_on_exit() {
stop_rss_service
stop_ping_monitor
rm_input
}
@@ -261,6 +286,7 @@ while true; do
# Start ping monitor for this connection
start_ping_monitor
start_rss_service
tail -f "$input" | socat -,ignoreeof "$socatAddress" | while read -r result ; do
# Strip carriage return from IRC protocol (CRLF line endings)
@@ -471,6 +497,7 @@ while true; do
# If we reach here, the connection was dropped
# Stop the ping monitor for this connection
stop_rss_service
stop_ping_monitor
# Check if this was an intentional exit
+22
View File
@@ -0,0 +1,22 @@
# Enable or disable RSS tracking.
enabled=true
# Poll interval in seconds. Default is one hour.
pollInterval=3600
# Delay between announcing multiple new RSS items from the same poll.
delayInterval=60
# Maximum number of new items announced per poll. Extra items are marked read.
maxItemsPerPoll=5
# Delay before the first poll after the IRC connection starts.
startupDelay=15
# User agent sent to RSS servers.
userAgent="stormbot-rss/1.0"
# Feed URLs to monitor. Real feed URLs should not contain literal spaces.
urls=(
"https://stormux.org/index.xml"
)
+549
View File
@@ -0,0 +1,549 @@
#!/usr/bin/env bash
if [[ $# -lt 4 ]]; then
echo "Usage: $0 <input-file> <log-file> <date-format> <channel> [channel...]" >&2
exit 1
fi
inputFile="$1"
logFile="$2"
dateFormat="$3"
shift 3
scriptPath="${BASH_SOURCE[0]}"
serviceDir="${scriptPath%/*}"
if [[ "$serviceDir" == "$scriptPath" ]]; then
serviceDir="."
fi
configFile="${serviceDir}/rss.conf"
stateFile="${serviceDir}/rss.seen"
enabled=true
pollInterval=3600
delayInterval=60
maxItemsPerPoll=5
startupDelay=15
userAgent="stormbot-rss/1.0"
urls=("https://stormux.org/index.xml")
channels=()
loadedUrls=()
pendingTitles=()
pendingLinks=()
itemsObserved=0
declare -A seenItems=()
trim_value() {
local value="$1"
value="${value#"${value%%[![:space:]]*}"}"
value="${value%"${value##*[![:space:]]}"}"
printf '%s' "$value"
}
strip_wrapping_quotes() {
local value="$1"
local firstChar
local lastChar
if [[ ${#value} -lt 2 ]]; then
printf '%s' "$value"
return
fi
firstChar="${value:0:1}"
lastChar="${value: -1}"
if [[ "$firstChar" == "$lastChar" && ( "$firstChar" == "\"" || "$firstChar" == "'" ) ]]; then
value="${value:1:${#value}-2}"
fi
printf '%s' "$value"
}
log_rss() {
local message="$1"
local timestamp
timestamp="$(date "+$dateFormat" 2>/dev/null || date)"
printf 'RSS service: %s [%s]\n' "$message" "$timestamp" >> "$logFile"
}
append_url_token() {
local url="$1"
url="$(trim_value "$url")"
url="${url%,}"
url="$(strip_wrapping_quotes "$url")"
if [[ -z "$url" ]]; then
return
fi
if [[ "$url" =~ ^https?://[^[:space:]]+$ ]]; then
loadedUrls+=("$url")
else
log_rss "Ignoring invalid feed URL: $url"
fi
}
parse_url_tokens() {
local content="$1"
local token
local doubleQuoteRegex='^[[:space:]]*"([^"]*)"(.*)$'
local singleQuoteRegex="^[[:space:]]*'([^']*)'(.*)$"
local closeRegex='^[[:space:]]*\)(.*)$'
local bareRegex='^[[:space:]]*([^[:space:])]+)(.*)$'
while [[ -n "$content" ]]; do
if [[ "$content" =~ $closeRegex ]]; then
break
elif [[ "$content" =~ $doubleQuoteRegex ]]; then
token="${BASH_REMATCH[1]}"
content="${BASH_REMATCH[2]}"
elif [[ "$content" =~ $singleQuoteRegex ]]; then
token="${BASH_REMATCH[1]}"
content="${BASH_REMATCH[2]}"
elif [[ "$content" =~ $bareRegex ]]; then
token="${BASH_REMATCH[1]}"
content="${BASH_REMATCH[2]}"
else
break
fi
append_url_token "$token"
done
}
assign_numeric_config() {
local key="$1"
local value="$2"
local numericValue
value="$(trim_value "${value%%#*}")"
if [[ ! "$value" =~ ^[0-9]+$ ]]; then
log_rss "Ignoring invalid numeric config ${key}=${value}"
return
fi
numericValue=$((10#$value))
case "$key" in
pollInterval)
if (( numericValue > 0 )); then
pollInterval="$numericValue"
else
log_rss "Ignoring pollInterval=0"
fi
;;
delayInterval)
delayInterval="$numericValue"
;;
maxItemsPerPoll)
if (( numericValue > 0 )); then
maxItemsPerPoll="$numericValue"
else
log_rss "Ignoring maxItemsPerPoll=0"
fi
;;
startupDelay)
startupDelay="$numericValue"
;;
esac
}
load_config() {
local rawLine
local line
local key
local value
local inUrls=false
local urlsConfigured=false
if [[ ! -f "$configFile" ]]; then
return
fi
loadedUrls=()
while IFS= read -r rawLine || [[ -n "$rawLine" ]]; do
line="$(trim_value "$rawLine")"
if [[ -z "$line" || "${line:0:1}" == "#" ]]; then
continue
fi
if [[ "$inUrls" == "true" ]]; then
parse_url_tokens "$line"
if [[ "$line" == *")"* ]]; then
inUrls=false
fi
continue
fi
if [[ "$line" != *"="* ]]; then
log_rss "Ignoring invalid config line: $line"
continue
fi
key="$(trim_value "${line%%=*}")"
value="$(trim_value "${line#*=}")"
if [[ "$key" == "urls" ]]; then
urlsConfigured=true
if [[ "$value" == \(* ]]; then
value="${value#(}"
parse_url_tokens "$value"
if [[ "$value" != *")"* ]]; then
inUrls=true
fi
else
log_rss "Ignoring invalid urls assignment"
fi
continue
fi
case "$key" in
enabled)
value="$(strip_wrapping_quotes "$(trim_value "${value%%#*}")")"
case "${value,,}" in
true|false)
enabled="${value,,}"
;;
*)
log_rss "Ignoring invalid enabled=${value}"
;;
esac
;;
pollInterval|delayInterval|maxItemsPerPoll|startupDelay)
assign_numeric_config "$key" "$value"
;;
userAgent)
value="$(strip_wrapping_quotes "$value")"
if [[ -n "$value" ]]; then
userAgent="$value"
fi
;;
*)
log_rss "Ignoring unknown config key: $key"
;;
esac
done < "$configFile"
if [[ "$urlsConfigured" == "true" ]]; then
urls=("${loadedUrls[@]}")
fi
}
normalize_channels() {
local rawChannel
local channelName
channels=()
for rawChannel in "$@"; do
channelName="$rawChannel"
if [[ "$channelName" != \#* ]]; then
channelName="#$channelName"
fi
if [[ "$channelName" =~ ^#[a-zA-Z0-9_-]+$ ]]; then
channels+=("$channelName")
else
log_rss "Skipping invalid channel name: $rawChannel"
fi
done
}
check_dependencies() {
local missing=()
local dependency
for dependency in curl xmlstarlet date grep sort tr; do
if ! command -v "$dependency" &>/dev/null; then
missing+=("$dependency")
fi
done
if [[ ${#missing[@]} -gt 0 ]]; then
log_rss "Missing dependencies: ${missing[*]}"
return 1
fi
return 0
}
load_seen_state() {
local itemId
seenItems=()
if [[ ! -f "$stateFile" ]]; then
return
fi
while IFS= read -r itemId || [[ -n "$itemId" ]]; do
if [[ -n "$itemId" ]]; then
seenItems["$itemId"]=1
fi
done < "$stateFile"
}
save_seen_state() {
local tempFile
tempFile="$(mktemp)" || return 1
if [[ ${#seenItems[@]} -gt 0 ]]; then
printf '%s\n' "${!seenItems[@]}" | sort > "$tempFile"
else
: > "$tempFile"
fi
mv "$tempFile" "$stateFile"
}
clean_text() {
local value="$1"
value="${value//$'\r'/ }"
value="${value//$'\n'/ }"
value="${value//$'\t'/ }"
value="${value//[$'\001'-$'\037'$'\177']/}"
value="$(trim_value "$value")"
printf '%s' "$value"
}
fetch_feed() {
local feedUrl="$1"
local feedFile="$2"
local errorFile
local errorMessage
errorFile="$(mktemp)" || return 1
if ! curl --fail --location --silent --show-error --proto '=http,https' --proto-redir '=http,https' --connect-timeout 5 --max-time 20 --user-agent "$userAgent" --output "$feedFile" "$feedUrl" 2>"$errorFile"; then
errorMessage="$(tr '\n' ' ' < "$errorFile")"
log_rss "Fetch failed for $feedUrl: $errorMessage"
rm -f "$errorFile"
return 1
fi
rm -f "$errorFile"
return 0
}
feed_has_doctype() {
local feedFile="$1"
LC_ALL=C grep -a -q '<!DOCTYPE' "$feedFile"
}
parse_feed() {
local feedFile="$1"
local fieldSeparator=$'\t'
local itemXPath='/*[local-name()="rss"]/*[local-name()="channel"]/*[local-name()="item"] | /*[local-name()="RDF"]/*[local-name()="item"] | /*[local-name()="feed"]/*[local-name()="entry"]'
xmlstarlet sel -T -t \
-m "$itemXPath" \
-v 'normalize-space(*[local-name()="title"][1])' -o "$fieldSeparator" \
-v 'normalize-space((*[local-name()="guid"][1] | *[local-name()="id"][1])[1])' -o "$fieldSeparator" \
-v 'normalize-space(*[local-name()="link"][@rel="alternate"][1]/@href)' -o "$fieldSeparator" \
-v 'normalize-space(*[local-name()="link"][1]/@href)' -o "$fieldSeparator" \
-v 'normalize-space(*[local-name()="link"][1])' -n \
"$feedFile"
}
queue_announcement() {
local title="$1"
local link="$2"
pendingTitles+=("$title")
pendingLinks+=("$link")
}
process_feed() {
local feedUrl="$1"
local firstRun="$2"
local feedFile
local parsedFile
local errorFile
local parseError
local title
local itemGuid
local alternateLink
local firstHref
local linkText
local link
local itemKey
feedFile="$(mktemp)" || return 1
parsedFile="$(mktemp)" || {
rm -f "$feedFile"
return 1
}
errorFile="$(mktemp)" || {
rm -f "$feedFile" "$parsedFile"
return 1
}
if ! fetch_feed "$feedUrl" "$feedFile"; then
rm -f "$feedFile" "$parsedFile" "$errorFile"
return 1
fi
if feed_has_doctype "$feedFile"; then
log_rss "Rejecting feed with DOCTYPE: $feedUrl"
rm -f "$feedFile" "$parsedFile" "$errorFile"
return 1
fi
if ! parse_feed "$feedFile" > "$parsedFile" 2>"$errorFile"; then
parseError="$(tr '\n' ' ' < "$errorFile")"
log_rss "Parse failed for $feedUrl: $parseError"
rm -f "$feedFile" "$parsedFile" "$errorFile"
return 1
fi
while IFS=$'\t' read -r title itemGuid alternateLink firstHref linkText || [[ -n "$title$itemGuid$alternateLink$firstHref$linkText" ]]; do
title="$(clean_text "$title")"
itemGuid="$(clean_text "$itemGuid")"
alternateLink="$(clean_text "$alternateLink")"
firstHref="$(clean_text "$firstHref")"
linkText="$(clean_text "$linkText")"
link="${alternateLink:-${firstHref:-$linkText}}"
if [[ -z "$title" ]]; then
title="$link"
fi
if [[ -z "$title" ]]; then
continue
fi
itemKey="${feedUrl}|${itemGuid:-${link:-$title}}"
((itemsObserved++))
if [[ -z "${seenItems[$itemKey]+seen}" ]]; then
seenItems["$itemKey"]=1
if [[ "$firstRun" == "false" ]]; then
queue_announcement "$title" "$link"
fi
fi
done < "$parsedFile"
rm -f "$feedFile" "$parsedFile" "$errorFile"
}
format_announcement() {
local title="$1"
local link="$2"
local message
if [[ -n "$link" ]]; then
message="RSS: ${title} - ${link}"
else
message="RSS: ${title}"
fi
if [[ ${#message} -gt 400 ]]; then
message="${message:0:397}..."
fi
printf '%s' "$message"
}
send_message() {
local channelName="$1"
local message="$2"
message="$(clean_text "$message")"
printf 'PRIVMSG %s :%s\r\n' "$channelName" "$message" >> "$inputFile"
}
deliver_announcements() {
local totalCount="${#pendingTitles[@]}"
local deliverCount="$totalCount"
local itemIndex
local channelName
local message
if (( totalCount == 0 )); then
return
fi
if (( deliverCount > maxItemsPerPoll )); then
deliverCount="$maxItemsPerPoll"
log_rss "Suppressing $((totalCount - deliverCount)) RSS items over maxItemsPerPoll"
fi
for ((itemIndex = 0; itemIndex < deliverCount; itemIndex++)); do
message="$(format_announcement "${pendingTitles[itemIndex]}" "${pendingLinks[itemIndex]}")"
for channelName in "${channels[@]}"; do
send_message "$channelName" "$message"
done
if (( itemIndex < deliverCount - 1 && delayInterval > 0 )); then
sleep "$delayInterval"
fi
done
}
poll_feeds() {
local firstRun="$1"
local feedUrl
pendingTitles=()
pendingLinks=()
itemsObserved=0
load_seen_state
for feedUrl in "${urls[@]}"; do
if [[ -n "$feedUrl" ]]; then
process_feed "$feedUrl" "$firstRun"
fi
done
if [[ "$firstRun" == "true" && "$itemsObserved" -eq 0 ]]; then
log_rss "Initial poll found no RSS items; leaving state uninitialized"
return
fi
save_seen_state
if [[ "$firstRun" == "true" ]]; then
log_rss "Initial poll marked $itemsObserved RSS items read"
return
fi
deliver_announcements
}
load_config
normalize_channels "$@"
if [[ "$enabled" != "true" ]]; then
log_rss "RSS service disabled"
exit 0
fi
if [[ ${#channels[@]} -eq 0 ]]; then
log_rss "No valid channels configured for RSS service"
exit 0
fi
if [[ ! -w "$inputFile" ]]; then
log_rss "Input file is not writable: $inputFile"
exit 1
fi
if ! check_dependencies; then
exit 1
fi
if (( startupDelay > 0 )); then
sleep "$startupDelay"
fi
while true; do
if [[ -f "$stateFile" ]]; then
poll_feeds false
else
poll_feeds true
fi
sleep "$pollInterval"
done