Generate Stable Database #229
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate Stable Database | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 1 */2 * *' | |
| jobs: | |
| generate: | |
| runs-on: ubuntu-latest | |
| permissions: write-all | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Dependency Setup | |
| run: | | |
| sudo apt-get update -y | |
| sudo apt install maven unzip git dos2unix -y | |
| - name: Start ClamAV daemon clamd | |
| uses: toblux/start-clamd-github-action@v0.2.1 | |
| with: | |
| db_main: true | |
| db_daily: true | |
| - name: Organize Directories & Static Sources | |
| run: | | |
| mkdir -p "${{ github.workspace }}/raw/" | |
| mkdir -p "${{ github.workspace }}/exclusions/" | |
| mkdir -p "${{ github.workspace }}/various/" | |
| mkdir -p "${{ github.workspace }}/domains/" | |
| cp ${{ github.workspace }}/static/* "${{ github.workspace }}/raw/" | |
| - name: DATABASE PREPARATION - ClamAV | |
| run: | | |
| mkdir /tmp/mss | |
| cd /tmp/mss | |
| cp /var/lib/clamav/main.c*d . | |
| cp /var/lib/clamav/daily.c*d . | |
| sigtool -u main.c*d | |
| sigtool -u daily.c*d | |
| mv /tmp/mss/*.fp* "${{ github.workspace }}/exclusions/" | |
| mv /tmp/mss/*.hsb "${{ github.workspace }}/raw/" | |
| mv /tmp/mss/*.hdb "${{ github.workspace }}/raw/" | |
| mv /tmp/mss/*.hdu "${{ github.workspace }}/raw/" | |
| mv /tmp/mss/*.hsu "${{ github.workspace }}/raw/" | |
| - name: DATABASE PREPARATION - ESET | |
| run: | | |
| git clone https://github.com/eset/malware-ioc "${{ github.workspace }}/various/eset-sources/" | |
| processHashes() { | |
| local name=$(basename $1); | |
| if [ -f $1/samples.$2 ]; then | |
| dos2unix $1/samples.$2 | |
| while IFS= read -r line | |
| do | |
| echo "$line" >> "${{ github.workspace }}/various/eset.$2"; | |
| done < "$1/samples.$2"; | |
| fi; | |
| } | |
| export -f processHashes; | |
| find "${{ github.workspace }}/various/eset-sources/" -maxdepth 2 -mindepth 1 -type d -exec bash -c 'processHashes "{}" md5 hdb' \; | |
| find "${{ github.workspace }}/various/eset-sources/" -maxdepth 2 -mindepth 1 -type d -exec bash -c 'processHashes "{}" sha1 hsb' \; | |
| find "${{ github.workspace }}/various/eset-sources/" -maxdepth 2 -mindepth 1 -type d -exec bash -c 'processHashes "{}" sha256 hsb' \; | |
| cp "${{ github.workspace }}/various/eset.md5" "${{ github.workspace }}/raw" | |
| cp "${{ github.workspace }}/various/eset.sha1" "${{ github.workspace }}/raw" | |
| cp "${{ github.workspace }}/various/eset.sha256" "${{ github.workspace }}/raw" | |
| - name: DATABASE PREPARATION - Signature-Base | |
| run: | | |
| wget https://raw.githubusercontent.com/Neo23x0/signature-base/refs/heads/master/iocs/hash-iocs.txt -O "${{ github.workspace }}/raw/signature-base.loki" | |
| - name: DATABASE PREPARATION - ThreatView | |
| run: | | |
| wget "https://threatview.io/Downloads/MD5-HASH-ALL.txt" -O - | sed 's/MD5 of //' >> "${{ github.workspace }}/various/threatview.md5" | |
| wget "https://threatview.io/Downloads/SHA-HASH-FEED.txt" -O - >> "${{ github.workspace }}/various/threatview.sha1" | |
| sort -u -o "${{ github.workspace }}/raw/threatview.md5" "${{ github.workspace }}/various/threatview.md5" | |
| sort -u -o "${{ github.workspace }}/raw/threatview.sha1" "${{ github.workspace }}/various/threatview.sha1" | |
| - name: DATABASE PREPARATION - ThreatFox | |
| run: | | |
| wget "https://threatfox.abuse.ch/export/csv/sha256/full/" -O "${{ github.workspace }}/various/download_threatfox.zip" | |
| unzip "${{ github.workspace }}/various/download_threatfox.zip" -d "${{ github.workspace }}/various/" | |
| tail -n +10 "${{ github.workspace }}/various/full_sha256.csv" | awk '{ print $4 } ' | sed 's/^"//' | sed 's/",$//' > "${{ github.workspace }}/raw/threatfox.sha256" | |
| - name: DATABASE PREPARATION - MalwareBazaar | |
| run: | | |
| wget "https://bazaar.abuse.ch/export/txt/sha256/full/" -O "${{ github.workspace }}/various/download_bazaar.zip" | |
| unzip "${{ github.workspace }}/various/download_bazaar.zip" -d "${{ github.workspace }}/various/" | |
| mv "${{ github.workspace }}/various/full_sha256.txt" "${{ github.workspace }}/raw/malware_bazaar.sha256" | |
| - name: DATABASE PREPARATION - CyberCure | |
| run: | | |
| wget "https://api.cybercure.ai/feed/get_hash?type=csv" -O - | sed 's/,/\n/g' >> "${{ github.workspace }}/various/cybercure.md5" | |
| sort -u -o "${{ github.workspace }}/raw/cybercure.md5" "${{ github.workspace }}/various/cybercure.md5" | |
| - name: DATABASE PREPARATION - Stalkerware | |
| run: | | |
| wget "https://raw.githubusercontent.com/AssoEchap/stalkerware-indicators/refs/heads/master/samples.csv" -O "${{ github.workspace }}/various/stalkerware.csv" | |
| while IFS=, read -r col1SHA col2Package col3Certificate col4Version col5Name | |
| do | |
| if [ -n "$col1SHA" ] && [ -n "$col5Name" ]; then | |
| echo "$col1SHA:0:$col5Name" >> "${{ github.workspace }}/raw/stalkerware.hsb"; | |
| fi; | |
| done < "${{ github.workspace }}/various/stalkerware.csv"; | |
| sed -i '1d' "${{ github.workspace }}/raw/stalkerware.hsb"; | |
| - name: DATABASE PREPARATION - Sanesecurity | |
| run: | | |
| mkdir -p "${{ github.workspace }}/various/sanesecurity-real/" | |
| cd "${{ github.workspace }}/various/sanesecurity-real/" | |
| rsync -av rsync://rsync.sanesecurity.net/sanesecurity . | |
| for f in *.hsb; do | |
| echo "" >> "${{ github.workspace }}/raw/sanesecurity-$f"; | |
| cat "$f" >> "${{ github.workspace }}/raw/sanesecurity-$f"; | |
| sort -u -o "${{ github.workspace }}/raw/sanesecurity-$f" "${{ github.workspace }}/raw/sanesecurity-$f"; | |
| done | |
| for f in *.hdb; do | |
| echo "" >> "${{ github.workspace }}/raw/sanesecurity-$f"; | |
| cat "$f" >> "${{ github.workspace }}/raw/sanesecurity-$f"; | |
| sort -u -o "${{ github.workspace }}/raw/sanesecurity-$f" "${{ github.workspace }}/raw/sanesecurity-$f"; | |
| done | |
| rm -v ${{ github.workspace }}/raw/sanesecurity-crdfam.clamav.hdb ${{ github.workspace }}/raw/sanesecurity-doppelstern.hdb ${{ github.workspace }}/raw/sanesecurity-malware.expert.hdb | |
| - name: DATABASE PREPERATION - Avast Covid-19 IOC | |
| run: | | |
| cd "${{ github.workspace }}/various" | |
| git clone https://github.com/avast/covid-19-ioc.git | |
| cd "covid-19-ioc/datasets/android" | |
| tail -n +2 *.csv | sed 's/,/ , /' | awk '{ print $1 }' | sort -u >> "${{ github.workspace }}/raw/avast-covid19.sha256" | |
| - name: DATABASE PREPERATION - Targeted Threats | |
| run: | | |
| cd "${{ github.workspace }}/various" | |
| git clone https://github.com/botherder/targetedthreats.git | |
| cd targetedthreats | |
| while IFS=, read -r col1MD5 col2SHA256 col3Family col4Actor col5Country col6Report | |
| do | |
| col4Actor=$(sed -e 's/^"//' -e 's/"$//' <<< "$col4Actor"); | |
| col3Family=$(sed -e 's/^"//' -e 's/"$//' <<< "$col3Family"); | |
| if [ -n "$col4Actor" ] && [ -n "$col3Family" ]; then | |
| description="$col4Actor-$col3Family"; | |
| elif [ -n "$col4Actor" ] && [ -z "$col3Family" ]; then | |
| description="$col4Actor"; | |
| elif [ -z "$col4Actor" ] && [ -n "$col3Family" ]; then | |
| description="$col3Family"; | |
| fi; | |
| if [ -z "$description" ]; then | |
| description="targetedthreats"; | |
| fi; | |
| col1MD5=$(sed -e 's/^"//' -e 's/"$//' <<< "$col1MD5"); | |
| if [ -n "$col1MD5" ]; then | |
| echo "$col1MD5:0:$description" >> ${{ github.workspace }}/raw/targetedthreats.hdb; | |
| fi; | |
| col2SHA256=$(sed -e 's/^"//' -e 's/"$//' <<< "$col2SHA256"); | |
| if [ -n "$col2SHA256" ]; then | |
| echo "$col2SHA256:0:$description" >> ${{ github.workspace }}/raw/targetedthreats.hsb; | |
| fi; | |
| done < samples.csv; | |
| sed -i '1d' ${{ github.workspace }}/raw/targetedthreats.hdb; | |
| sed -i '1d' ${{ github.workspace }}/raw/targetedthreats.hsb; | |
| - name: DATABASE PREPERATION - Divest Domains | |
| run: wget "divested.dev/hosts-domains" -O "${{ github.workspace }}/domains/divested.txt" | |
| - name: GENERATE HYPATIA SIGNATURE DATABASES | |
| run: | | |
| mvn -q -B --no-transfer-progress clean compile exec:java -Dexec.mainClass="org.maintainteam.hypatiadatabases.App" -Dexec.args="${{ github.workspace }}/raw/ ${{ github.workspace }}/domains/divested.txt" > "${{ github.workspace }}/various/generation_report.txt" | |
| cat "${{ github.workspace }}/various/generation_report.txt" | |
| echo "<pre>" >> "${{ github.workspace }}/various/generation_report.html" | |
| cat "${{ github.workspace }}/various/generation_report.txt" >> "${{ github.workspace }}/various/generation_report.html" | |
| echo "</pre>" >> "${{ github.workspace }}/various/generation_report.html" | |
| echo "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">"${{ github.workspace }}/various/database_generation_ci.txt" | |
| - name: Debug Info | |
| run: | | |
| ls -hla "${{ github.workspace }}/various/" | |
| ls -hla "${{ github.workspace }}/raw/" | |
| ls -hla "${{ github.workspace }}/exclusions/" | |
| ls -hla "${{ github.workspace }}/domains/" | |
| tree -a "${{ github.workspace }}" | |
| - name: Select files to use | |
| run: | | |
| mkdir -p "${{ github.workspace }}/deploy" | |
| cp "${{ github.workspace }}/raw/hypatia-domains-bloom.bin" "${{ github.workspace }}/deploy" | |
| cp "${{ github.workspace }}/raw/hypatia-md5-bloom.bin" "${{ github.workspace }}/deploy" | |
| cp "${{ github.workspace }}/raw/hypatia-sha1-bloom.bin" "${{ github.workspace }}/deploy" | |
| cp "${{ github.workspace }}/raw/hypatia-sha256-bloom.bin" "${{ github.workspace }}/deploy" | |
| cp "${{ github.workspace }}/various/generation_report.html" "${{ github.workspace }}/deploy" | |
| cp "${{ github.workspace }}/various/database_generation_ci.txt" "${{ github.workspace }}/deploy" | |
| cp "${{ github.workspace }}/static/template.html" "${{ github.workspace }}/deploy" | |
| date > "${{ github.workspace }}/deploy/generation_time.html" | |
| - name: Deploy to unsigned branch | |
| uses: peaceiris/actions-gh-pages@v4 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: "${{ github.workspace }}/deploy" | |
| force_orphan: true | |
| publish_branch: unsigned |