Storm Dragon
550 lines
13 KiB
Bash
Executable File
550 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
if [[ $# -lt 4 ]]; then
|
|
echo "Usage: $0 <input-file> <log-file> <date-format> <channel> [channel...]" >&2
|
|
exit 1
|
|
fi
|
|
|
|
inputFile="$1"
|
|
logFile="$2"
|
|
dateFormat="$3"
|
|
shift 3
|
|
|
|
scriptPath="${BASH_SOURCE[0]}"
|
|
serviceDir="${scriptPath%/*}"
|
|
if [[ "$serviceDir" == "$scriptPath" ]]; then
|
|
serviceDir="."
|
|
fi
|
|
|
|
configFile="${serviceDir}/rss.conf"
|
|
stateFile="${serviceDir}/rss.seen"
|
|
|
|
enabled=true
|
|
pollInterval=3600
|
|
delayInterval=60
|
|
maxItemsPerPoll=5
|
|
startupDelay=15
|
|
userAgent="stormbot-rss/1.0"
|
|
urls=("https://stormux.org/index.xml")
|
|
channels=()
|
|
loadedUrls=()
|
|
pendingTitles=()
|
|
pendingLinks=()
|
|
itemsObserved=0
|
|
declare -A seenItems=()
|
|
|
|
trim_value() {
|
|
local value="$1"
|
|
value="${value#"${value%%[![:space:]]*}"}"
|
|
value="${value%"${value##*[![:space:]]}"}"
|
|
printf '%s' "$value"
|
|
}
|
|
|
|
strip_wrapping_quotes() {
|
|
local value="$1"
|
|
local firstChar
|
|
local lastChar
|
|
|
|
if [[ ${#value} -lt 2 ]]; then
|
|
printf '%s' "$value"
|
|
return
|
|
fi
|
|
|
|
firstChar="${value:0:1}"
|
|
lastChar="${value: -1}"
|
|
if [[ "$firstChar" == "$lastChar" && ( "$firstChar" == "\"" || "$firstChar" == "'" ) ]]; then
|
|
value="${value:1:${#value}-2}"
|
|
fi
|
|
|
|
printf '%s' "$value"
|
|
}
|
|
|
|
log_rss() {
|
|
local message="$1"
|
|
local timestamp
|
|
|
|
timestamp="$(date "+$dateFormat" 2>/dev/null || date)"
|
|
printf 'RSS service: %s [%s]\n' "$message" "$timestamp" >> "$logFile"
|
|
}
|
|
|
|
append_url_token() {
|
|
local url="$1"
|
|
|
|
url="$(trim_value "$url")"
|
|
url="${url%,}"
|
|
url="$(strip_wrapping_quotes "$url")"
|
|
|
|
if [[ -z "$url" ]]; then
|
|
return
|
|
fi
|
|
|
|
if [[ "$url" =~ ^https?://[^[:space:]]+$ ]]; then
|
|
loadedUrls+=("$url")
|
|
else
|
|
log_rss "Ignoring invalid feed URL: $url"
|
|
fi
|
|
}
|
|
|
|
parse_url_tokens() {
|
|
local content="$1"
|
|
local token
|
|
local doubleQuoteRegex='^[[:space:]]*"([^"]*)"(.*)$'
|
|
local singleQuoteRegex="^[[:space:]]*'([^']*)'(.*)$"
|
|
local closeRegex='^[[:space:]]*\)(.*)$'
|
|
local bareRegex='^[[:space:]]*([^[:space:])]+)(.*)$'
|
|
|
|
while [[ -n "$content" ]]; do
|
|
if [[ "$content" =~ $closeRegex ]]; then
|
|
break
|
|
elif [[ "$content" =~ $doubleQuoteRegex ]]; then
|
|
token="${BASH_REMATCH[1]}"
|
|
content="${BASH_REMATCH[2]}"
|
|
elif [[ "$content" =~ $singleQuoteRegex ]]; then
|
|
token="${BASH_REMATCH[1]}"
|
|
content="${BASH_REMATCH[2]}"
|
|
elif [[ "$content" =~ $bareRegex ]]; then
|
|
token="${BASH_REMATCH[1]}"
|
|
content="${BASH_REMATCH[2]}"
|
|
else
|
|
break
|
|
fi
|
|
|
|
append_url_token "$token"
|
|
done
|
|
}
|
|
|
|
assign_numeric_config() {
|
|
local key="$1"
|
|
local value="$2"
|
|
local numericValue
|
|
|
|
value="$(trim_value "${value%%#*}")"
|
|
if [[ ! "$value" =~ ^[0-9]+$ ]]; then
|
|
log_rss "Ignoring invalid numeric config ${key}=${value}"
|
|
return
|
|
fi
|
|
|
|
numericValue=$((10#$value))
|
|
case "$key" in
|
|
pollInterval)
|
|
if (( numericValue > 0 )); then
|
|
pollInterval="$numericValue"
|
|
else
|
|
log_rss "Ignoring pollInterval=0"
|
|
fi
|
|
;;
|
|
delayInterval)
|
|
delayInterval="$numericValue"
|
|
;;
|
|
maxItemsPerPoll)
|
|
if (( numericValue > 0 )); then
|
|
maxItemsPerPoll="$numericValue"
|
|
else
|
|
log_rss "Ignoring maxItemsPerPoll=0"
|
|
fi
|
|
;;
|
|
startupDelay)
|
|
startupDelay="$numericValue"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
load_config() {
|
|
local rawLine
|
|
local line
|
|
local key
|
|
local value
|
|
local inUrls=false
|
|
local urlsConfigured=false
|
|
|
|
if [[ ! -f "$configFile" ]]; then
|
|
return
|
|
fi
|
|
|
|
loadedUrls=()
|
|
while IFS= read -r rawLine || [[ -n "$rawLine" ]]; do
|
|
line="$(trim_value "$rawLine")"
|
|
|
|
if [[ -z "$line" || "${line:0:1}" == "#" ]]; then
|
|
continue
|
|
fi
|
|
|
|
if [[ "$inUrls" == "true" ]]; then
|
|
parse_url_tokens "$line"
|
|
if [[ "$line" == *")"* ]]; then
|
|
inUrls=false
|
|
fi
|
|
continue
|
|
fi
|
|
|
|
if [[ "$line" != *"="* ]]; then
|
|
log_rss "Ignoring invalid config line: $line"
|
|
continue
|
|
fi
|
|
|
|
key="$(trim_value "${line%%=*}")"
|
|
value="$(trim_value "${line#*=}")"
|
|
|
|
if [[ "$key" == "urls" ]]; then
|
|
urlsConfigured=true
|
|
if [[ "$value" == \(* ]]; then
|
|
value="${value#(}"
|
|
parse_url_tokens "$value"
|
|
if [[ "$value" != *")"* ]]; then
|
|
inUrls=true
|
|
fi
|
|
else
|
|
log_rss "Ignoring invalid urls assignment"
|
|
fi
|
|
continue
|
|
fi
|
|
|
|
case "$key" in
|
|
enabled)
|
|
value="$(strip_wrapping_quotes "$(trim_value "${value%%#*}")")"
|
|
case "${value,,}" in
|
|
true|false)
|
|
enabled="${value,,}"
|
|
;;
|
|
*)
|
|
log_rss "Ignoring invalid enabled=${value}"
|
|
;;
|
|
esac
|
|
;;
|
|
pollInterval|delayInterval|maxItemsPerPoll|startupDelay)
|
|
assign_numeric_config "$key" "$value"
|
|
;;
|
|
userAgent)
|
|
value="$(strip_wrapping_quotes "$value")"
|
|
if [[ -n "$value" ]]; then
|
|
userAgent="$value"
|
|
fi
|
|
;;
|
|
*)
|
|
log_rss "Ignoring unknown config key: $key"
|
|
;;
|
|
esac
|
|
done < "$configFile"
|
|
|
|
if [[ "$urlsConfigured" == "true" ]]; then
|
|
urls=("${loadedUrls[@]}")
|
|
fi
|
|
}
|
|
|
|
normalize_channels() {
|
|
local rawChannel
|
|
local channelName
|
|
|
|
channels=()
|
|
for rawChannel in "$@"; do
|
|
channelName="$rawChannel"
|
|
if [[ "$channelName" != \#* ]]; then
|
|
channelName="#$channelName"
|
|
fi
|
|
|
|
if [[ "$channelName" =~ ^#[a-zA-Z0-9_-]+$ ]]; then
|
|
channels+=("$channelName")
|
|
else
|
|
log_rss "Skipping invalid channel name: $rawChannel"
|
|
fi
|
|
done
|
|
}
|
|
|
|
check_dependencies() {
|
|
local missing=()
|
|
local dependency
|
|
|
|
for dependency in curl xmlstarlet date grep sort tr; do
|
|
if ! command -v "$dependency" &>/dev/null; then
|
|
missing+=("$dependency")
|
|
fi
|
|
done
|
|
|
|
if [[ ${#missing[@]} -gt 0 ]]; then
|
|
log_rss "Missing dependencies: ${missing[*]}"
|
|
return 1
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
load_seen_state() {
|
|
local itemId
|
|
|
|
seenItems=()
|
|
if [[ ! -f "$stateFile" ]]; then
|
|
return
|
|
fi
|
|
|
|
while IFS= read -r itemId || [[ -n "$itemId" ]]; do
|
|
if [[ -n "$itemId" ]]; then
|
|
seenItems["$itemId"]=1
|
|
fi
|
|
done < "$stateFile"
|
|
}
|
|
|
|
save_seen_state() {
|
|
local tempFile
|
|
|
|
tempFile="$(mktemp)" || return 1
|
|
if [[ ${#seenItems[@]} -gt 0 ]]; then
|
|
printf '%s\n' "${!seenItems[@]}" | sort > "$tempFile"
|
|
else
|
|
: > "$tempFile"
|
|
fi
|
|
|
|
mv "$tempFile" "$stateFile"
|
|
}
|
|
|
|
clean_text() {
|
|
local value="$1"
|
|
|
|
value="${value//$'\r'/ }"
|
|
value="${value//$'\n'/ }"
|
|
value="${value//$'\t'/ }"
|
|
value="${value//[$'\001'-$'\037'$'\177']/}"
|
|
value="$(trim_value "$value")"
|
|
printf '%s' "$value"
|
|
}
|
|
|
|
fetch_feed() {
|
|
local feedUrl="$1"
|
|
local feedFile="$2"
|
|
local errorFile
|
|
local errorMessage
|
|
|
|
errorFile="$(mktemp)" || return 1
|
|
if ! curl --fail --location --silent --show-error --proto '=http,https' --proto-redir '=http,https' --connect-timeout 5 --max-time 20 --user-agent "$userAgent" --output "$feedFile" "$feedUrl" 2>"$errorFile"; then
|
|
errorMessage="$(tr '\n' ' ' < "$errorFile")"
|
|
log_rss "Fetch failed for $feedUrl: $errorMessage"
|
|
rm -f "$errorFile"
|
|
return 1
|
|
fi
|
|
|
|
rm -f "$errorFile"
|
|
return 0
|
|
}
|
|
|
|
feed_has_doctype() {
|
|
local feedFile="$1"
|
|
|
|
LC_ALL=C grep -a -q '<!DOCTYPE' "$feedFile"
|
|
}
|
|
|
|
parse_feed() {
|
|
local feedFile="$1"
|
|
local fieldSeparator=$'\t'
|
|
local itemXPath='/*[local-name()="rss"]/*[local-name()="channel"]/*[local-name()="item"] | /*[local-name()="RDF"]/*[local-name()="item"] | /*[local-name()="feed"]/*[local-name()="entry"]'
|
|
|
|
xmlstarlet sel -T -t \
|
|
-m "$itemXPath" \
|
|
-v 'normalize-space(*[local-name()="title"][1])' -o "$fieldSeparator" \
|
|
-v 'normalize-space((*[local-name()="guid"][1] | *[local-name()="id"][1])[1])' -o "$fieldSeparator" \
|
|
-v 'normalize-space(*[local-name()="link"][@rel="alternate"][1]/@href)' -o "$fieldSeparator" \
|
|
-v 'normalize-space(*[local-name()="link"][1]/@href)' -o "$fieldSeparator" \
|
|
-v 'normalize-space(*[local-name()="link"][1])' -n \
|
|
"$feedFile"
|
|
}
|
|
|
|
queue_announcement() {
|
|
local title="$1"
|
|
local link="$2"
|
|
|
|
pendingTitles+=("$title")
|
|
pendingLinks+=("$link")
|
|
}
|
|
|
|
process_feed() {
|
|
local feedUrl="$1"
|
|
local firstRun="$2"
|
|
local feedFile
|
|
local parsedFile
|
|
local errorFile
|
|
local parseError
|
|
local title
|
|
local itemGuid
|
|
local alternateLink
|
|
local firstHref
|
|
local linkText
|
|
local link
|
|
local itemKey
|
|
|
|
feedFile="$(mktemp)" || return 1
|
|
parsedFile="$(mktemp)" || {
|
|
rm -f "$feedFile"
|
|
return 1
|
|
}
|
|
errorFile="$(mktemp)" || {
|
|
rm -f "$feedFile" "$parsedFile"
|
|
return 1
|
|
}
|
|
|
|
if ! fetch_feed "$feedUrl" "$feedFile"; then
|
|
rm -f "$feedFile" "$parsedFile" "$errorFile"
|
|
return 1
|
|
fi
|
|
|
|
if feed_has_doctype "$feedFile"; then
|
|
log_rss "Rejecting feed with DOCTYPE: $feedUrl"
|
|
rm -f "$feedFile" "$parsedFile" "$errorFile"
|
|
return 1
|
|
fi
|
|
|
|
if ! parse_feed "$feedFile" > "$parsedFile" 2>"$errorFile"; then
|
|
parseError="$(tr '\n' ' ' < "$errorFile")"
|
|
log_rss "Parse failed for $feedUrl: $parseError"
|
|
rm -f "$feedFile" "$parsedFile" "$errorFile"
|
|
return 1
|
|
fi
|
|
|
|
while IFS=$'\t' read -r title itemGuid alternateLink firstHref linkText || [[ -n "$title$itemGuid$alternateLink$firstHref$linkText" ]]; do
|
|
title="$(clean_text "$title")"
|
|
itemGuid="$(clean_text "$itemGuid")"
|
|
alternateLink="$(clean_text "$alternateLink")"
|
|
firstHref="$(clean_text "$firstHref")"
|
|
linkText="$(clean_text "$linkText")"
|
|
link="${alternateLink:-${firstHref:-$linkText}}"
|
|
|
|
if [[ -z "$title" ]]; then
|
|
title="$link"
|
|
fi
|
|
|
|
if [[ -z "$title" ]]; then
|
|
continue
|
|
fi
|
|
|
|
itemKey="${feedUrl}|${itemGuid:-${link:-$title}}"
|
|
((itemsObserved++))
|
|
|
|
if [[ -z "${seenItems[$itemKey]+seen}" ]]; then
|
|
seenItems["$itemKey"]=1
|
|
if [[ "$firstRun" == "false" ]]; then
|
|
queue_announcement "$title" "$link"
|
|
fi
|
|
fi
|
|
done < "$parsedFile"
|
|
|
|
rm -f "$feedFile" "$parsedFile" "$errorFile"
|
|
}
|
|
|
|
format_announcement() {
|
|
local title="$1"
|
|
local link="$2"
|
|
local message
|
|
|
|
if [[ -n "$link" ]]; then
|
|
message="${title} - ${link}"
|
|
else
|
|
message="RSS: ${title}"
|
|
fi
|
|
|
|
if [[ ${#message} -gt 400 ]]; then
|
|
message="${message:0:397}..."
|
|
fi
|
|
|
|
printf '%s' "$message"
|
|
}
|
|
|
|
send_message() {
|
|
local channelName="$1"
|
|
local message="$2"
|
|
|
|
message="$(clean_text "$message")"
|
|
printf 'PRIVMSG %s :%s\r\n' "$channelName" "$message" >> "$inputFile"
|
|
}
|
|
|
|
deliver_announcements() {
|
|
local totalCount="${#pendingTitles[@]}"
|
|
local deliverCount="$totalCount"
|
|
local itemIndex
|
|
local channelName
|
|
local message
|
|
|
|
if (( totalCount == 0 )); then
|
|
return
|
|
fi
|
|
|
|
if (( deliverCount > maxItemsPerPoll )); then
|
|
deliverCount="$maxItemsPerPoll"
|
|
log_rss "Suppressing $((totalCount - deliverCount)) RSS items over maxItemsPerPoll"
|
|
fi
|
|
|
|
for ((itemIndex = 0; itemIndex < deliverCount; itemIndex++)); do
|
|
message="$(format_announcement "${pendingTitles[itemIndex]}" "${pendingLinks[itemIndex]}")"
|
|
|
|
for channelName in "${channels[@]}"; do
|
|
send_message "$channelName" "$message"
|
|
done
|
|
|
|
if (( itemIndex < deliverCount - 1 && delayInterval > 0 )); then
|
|
sleep "$delayInterval"
|
|
fi
|
|
done
|
|
}
|
|
|
|
poll_feeds() {
|
|
local firstRun="$1"
|
|
local feedUrl
|
|
|
|
pendingTitles=()
|
|
pendingLinks=()
|
|
itemsObserved=0
|
|
load_seen_state
|
|
|
|
for feedUrl in "${urls[@]}"; do
|
|
if [[ -n "$feedUrl" ]]; then
|
|
process_feed "$feedUrl" "$firstRun"
|
|
fi
|
|
done
|
|
|
|
if [[ "$firstRun" == "true" && "$itemsObserved" -eq 0 ]]; then
|
|
log_rss "Initial poll found no RSS items; leaving state uninitialized"
|
|
return
|
|
fi
|
|
|
|
save_seen_state
|
|
|
|
if [[ "$firstRun" == "true" ]]; then
|
|
log_rss "Initial poll marked $itemsObserved RSS items read"
|
|
return
|
|
fi
|
|
|
|
deliver_announcements
|
|
}
|
|
|
|
load_config
|
|
normalize_channels "$@"
|
|
|
|
if [[ "$enabled" != "true" ]]; then
|
|
log_rss "RSS service disabled"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ ${#channels[@]} -eq 0 ]]; then
|
|
log_rss "No valid channels configured for RSS service"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ ! -w "$inputFile" ]]; then
|
|
log_rss "Input file is not writable: $inputFile"
|
|
exit 1
|
|
fi
|
|
|
|
if ! check_dependencies; then
|
|
exit 1
|
|
fi
|
|
|
|
if (( startupDelay > 0 )); then
|
|
sleep "$startupDelay"
|
|
fi
|
|
|
|
while true; do
|
|
if [[ -f "$stateFile" ]]; then
|
|
poll_feeds false
|
|
else
|
|
poll_feeds true
|
|
fi
|
|
|
|
sleep "$pollInterval"
|
|
done
|