#/bin/sh
# Guy Bruneau, guybruneau@outlook.com
# Date: 31 May 2016
# Version: 0.3
# Date: 23 February 2022
# Version: 0.4
# Title: parsing_snort_sid.sh
# Source code put in public domain by Guy Bruneau, no Copyright
# Use at your own risk
# This software is provided "as is" without express or implied warranty or support
# Added URL for SID, References and CVE
# Added 3 methods to select the rules based on Sourcefire
# rulestates-balanced-ips.states, rulestates-connectivity-ips.states
# rulestates-max-detect-ips.states, rulestates-no-rules-active.states
# rulestates-security-ips.states
# NetWitness decoder configuration
# https://community.rsa.com/t5/netwitness-platform-online/snort-parsers/ta-p/621026
# See parsers.options to add HTTP="decompress=true" Snort="udm=true" GeoIP2="ipaddr=ip.src,ip.dst,ipv6.src,ipv6.dst"
#---------------------------------------
# This script is used to parse Snort signatures into single text files. The filename
# being the Signature ID (SID) for easy access via a web browser.
# The true location of the Snort signatures is:
# /var/www/html/sid
# mkdir /var/www/html/sid
# On SA server: /opt/rsa/nw-ui/html/sid
# Modify the location of the webpage on the SA server (i.e. change 192.168.25.30)
### Set some variables
NOW=$(date "+%d-%m-%Y")
WEBPAGE="192.168.25.30"
RULEDIR="/home/guy/snort"
REMOVESID="/root/scripts/remove_sid.txt"
DECODER=/root/scripts/decoder_list.txt
OUTPUTDIR="/opt/rsa/nw-ui/html/sid"
NWADMIN="admin"
NWPASS="netwitness"
# Set color list
RED='\033[0;31m' # Red
DGRAY='\033[0;30m' # Dark Gray
GREEN='\033[0;32m' # Green
YELLOW='\033[0;33m' # Yellow
BLUE='\033[0;34m' # Blue
LGREEN='\033[1;32m' # Light Green
NC='\033[0m' # No color
# Latest snort ruleset
# File to remove false positive or un-needed SID from alert
# Check if the file exist
if [[ ! -f $REMOVESID ]]; then
touch $REMOVESID
fi
# File that contains a list of all Decoders
# Check if the file exist
if [[ ! -f $DECODER ]]; then
touch $DECODER
fi
# Check if the decoder file is empty, if empty stop
if [[ ! -s $DECODER ]]; then
clear
echo
echo -e "---> ${GREEN}Decoder file with IPs is empty, add decoders!${NC}"
echo
echo
exit
fi
# Check to make sure web directory exists
#OUTPUTDIR="/var/www/html/sid"
if [[ ! -d $OUTPUTDIR ]]; then
mkdir -p $OUTPUTDIR
fi
# -------------------- Start of Script ---------------------
# Check to see if we have a Snort rule tarball
if [ ! -f $RULEDIR/snortrules-*.tar.gz ]; then
echo "File no found!"
exit 0
fi
# If we have a Snort rule tarball, then we continue with the update
# Change directory to where the ruleset is located
cd $RULEDIR
# List the name of the latest ruleset to include later in the webpage
RULESET=`ls -1`
# Extract the tarball
tar zxvf $RULEDIR/snortrules-*.tar.gz
# Start by removing all the current list of Snort signatures
#/bin/rm -f /var/www/html/sid/*
/bin/rm -f /opt/rsa/nw-ui/html/sid/*
# Extract each signatures and dump the result into an individual text file based
# on the SID.
# Only use the active rules from one of the following lists
#cat $RULEDIR/rules/rulestates-max-detect-ips.states | sed 's/#.*//g' | sed 's/sid: /sid:/g'| sed 's/.*sid:\([[:digit:]]\{1,6\}\);.*/sid:\1/g' | sort -n |uniq | tail -n +2 > $RULEDIR/sid_list
#cat $RULEDIR/rules/rulestates-security-ips.states | sed 's/#.*//g' | sed 's/sid: /sid:/g'| sed 's/.*sid:\([[:digit:]]\{1,6\}\);.*/sid:\1/g' | sort -n |uniq | tail -n +2 > $RULEDIR/sid_list
cat $RULEDIR/rules/rulestates-balanced-ips.states | sed 's/#.*//g' | sed 's/sid: /sid:/g'| sed 's/.*sid:\([[:digit:]]\{1,6\}\);.*/sid:\1/g' | sort -n |uniq | tail -n +2 > $RULEDIR/sid_list
# Make a list of list of active rules for parsing the webpage
/usr/bin/grep -Fwhf $RULEDIR/sid_list $RULEDIR/rules/snort3* > $RULEDIR/list.rules
sed -i 's/: )$/:)/g' $RULEDIR/list.rules
# Ensure that all SID are formatted correctly i.e. sid:12345
sed -i 's/\(.*sid:\)[[:space:]]\{1,\}\([[:digit:]]\{1,10\};.*\)/\1\2/g' $RULEDIR/list.rules
# Rename the comma separator in all content (,) to a full column separator (;)
# Rename fast_patter,nocase to fast_pattern:only
sed -i 's/",/"; /g; s/fast_pattern,nocase/fast_pattern:only/g' $RULEDIR/list.rules
# Count the number of rules and save the result
COUNTRULES=`wc -l $RULEDIR/list.rules | awk '{ print $1 }'`
# Load all the rules into an array for processing
array=(`cat $RULEDIR/list.rules | sed 's/[ ]/_/g' | grep -o -P '(?<=sid:)\d+'`)
cd $RULEDIR
touch LIST=${array[*]}
/bin/rm LIST=*
# Need a variable to take care of the semi-column in the grep
comma=';'
# Get all the signatures from the list.rules file and copy each rule in the correct
# sid file
i=1
for ((i=${#array[@]}; i>=0; i--)); do
FILE=${array[$i]}
#grep sid:${array[$i]}$comma $RULEDIR/list.rules > /var/www/html/sid/${array[$i]}
grep sid:${array[$i]}$comma $RULEDIR/list.rules > /opt/rsa/nw-ui/html/sid/${array[$i]}
done
# Add file extension .txt to each files
# and add a new line at the end of each line to separate each signatures
#cd /var/www/html/sid
cd /opt/rsa/nw-ui/html/sid
# Delete empty files before continuing
find . -type f -empty -print -delete
# Rename less than "<"
sed -i 's/\</g' *
for file in * ; do
mv ${file} ${file/%/.htm};
done
# These 3 sed commands add HTML code to CVE, SID and References
# Add URL to all CVE web reference
sed -i -e ':loop' -e 's/reference:cve,\([[:digit:]]\{1,4\}\)\(-[[:digit:]]\{1,5\}\)\(;.*\)/CVE-\1\2<\/a>\3/g' -e 't loop' *.htm
# Add URL to Snort SID reference
sed -i 's/sid:\([[:digit:]]\{1,5\}\)\(;.*\)/sid:\1<\/a>\2/g' *.htm
# Configure all reference,url to their URL
sed -i 's/\(reference:url,\)/\n\1/g; s/\(classtype:.*\)/\n\1/g' *.htm
sed -i 's/reference:url,\(.*\);/Reference<\/a>/g' *.htm
# Create Snort Signature web page
echo "Current ruleset is: $RULESET" > index.htm
echo "" >> index.htm
echo "Rules updated on: $NOW. There are $COUNTRULES rules in this ruleset!" >> index.htm
echo "" >> index.htm
ls -l | awk '{ print $9 }' | sed 's/.htm//g' | egrep -v "index*|^$" | sort -n | sed 's/\([[:digit:]]\{1,7\}\)/\1<\/a><\/br>/g' >> index.htm
# Remove these SID (signatures) from the rule file
for i in `cat /root/scripts/remove_sid.txt` ; do
sed -i 's/.*'$i'.*//g' $RULEDIR/list.rules
done
# Send the list.rules to the decoder(s)
# Using shared keys using ssh-keygen with no password
# Copy public key to decoders in /root/.ssh
# ssh-copy-id root@nwdec1
# Reload decoder parsers
# Create a list
for i in `cat /root/scripts/decoder_list.txt` ; do
/usr/bin/scp $RULEDIR/list.rules root@$i:/etc/netwitness/ng/parsers/snort
curl http://$NWADMIN:$NWPASS@$i:50104/decoder/parsers?msg=reload
done
# This query can be used to check if the new rules have been reloaded
#ssh nwdec1 '/usr/bin/grep "Loaded.*rules" /var/log/messages | tail -2'
# Remove the old list of rules
/bin/rm -fr $RULEDIR/*