From 61126b35771eaa7537757362f264dbc8b6a32ed7 Mon Sep 17 00:00:00 2001 From: Mathieu Deous Date: Fri, 15 Apr 2022 22:02:16 +0200 Subject: Rewrite shell script in Go --- .github/workflows/test.yml | 35 ++++ .gitignore | 2 + .travis.yml | 21 -- Makefile | 24 +-- README.md | 47 +++-- debian/changelog | 101 --------- debian/compat | 1 - debian/conffiles | 0 debian/control | 14 -- debian/copyright | 7 - debian/files | 1 - debian/nbs-phpmalwarefinder.dirs | 1 - debian/nbs-phpmalwarefinder.install | 12 -- debian/rules | 12 -- go.mod | 10 + go.sum | 7 + php-malware-finder/phpmalwarefinder | 96 --------- php-malware-finder/phpmalwarefinder.go | 372 +++++++++++++++++++++++++++++++++ php-malware-finder/tests.sh | 2 +- 19 files changed, 461 insertions(+), 304 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore delete mode 100644 .travis.yml delete mode 100644 debian/changelog delete mode 100644 debian/compat delete mode 100644 debian/conffiles delete mode 100644 debian/control delete mode 100644 debian/copyright delete mode 100644 debian/files delete mode 100644 debian/nbs-phpmalwarefinder.dirs delete mode 100644 debian/nbs-phpmalwarefinder.install delete mode 100755 debian/rules create mode 100644 go.mod create mode 100644 go.sum delete mode 100755 php-malware-finder/phpmalwarefinder create mode 100644 php-malware-finder/phpmalwarefinder.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..f29e422 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,35 @@ +name: Test Suite + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: '^1.17' + + # apt repos don't have YARA v4.2, install it from git + - name: Install YARA + run: | + git clone --depth 1 https://github.com/virustotal/yara.git + cd yara + bash ./build.sh + sudo make install + cd .. + + - name: Run tests + run: | + LD_LIBRARY_PATH=/usr/local/lib make tests diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..639d072 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +php-malware-finder/phpmalwarefinder +.idea diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 132447d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: c - -addons: - apt: - packages: - - devscripts - - fakeroot - - debhelper - -install: - - git clone --depth 1 https://github.com/plusvic/yara.git yara3 - - cd yara3 - - bash ./build.sh - - ./configure - - make - - cp ./yara ../php-malware-finder/ - - cd .. - -script: - - make tests - - make deb diff --git a/Makefile b/Makefile index 1fa1a91..931f4e7 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,15 @@ -VERSION=1.0 -DEBVER := $(shell sed 's,[/\.].*,,' < /etc/debian_version) +.PHONY: clean deps tests -tests: - @cd ./php-malware-finder && bash ./tests.sh +all: php-malware-finder/phpmalwarefinder -debclean: - rm -rf php-malware-finder/debian - rm -f *.build *.changes *.deb +php-malware-finder/phpmalwarefinder: + go build -o php-malware-finder/phpmalwarefinder php-malware-finder/phpmalwarefinder.go -extract: - cp -r debian php-malware-finder - git checkout php-malware-finder/php.yar +clean: + rm -f php-malware-finder/phpmalwarefinder -rpm: - @echo "no rpm build target for now, feel free to submit one" +deps: + go mod tidy -v -deb: debclean extract - cd php-malware-finder && debuild -b -us -uc --lintian-opts -X po-debconf --profile debian +tests: php-malware-finder/phpmalwarefinder + @cd ./php-malware-finder && bash ./tests.sh diff --git a/README.md b/README.md index 1b60ce1..6ae0b07 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/jvoisin/php-malware-finder.svg?branch=master)](https://travis-ci.org/jvoisin/php-malware-finder) +![Test Suite](https://github.com/jvoisin/php-malware-finder/actions/workflows/test.yml/badge.svg) # PHP Malware Finder @@ -54,38 +54,39 @@ Detection is performed by crawling the filesystem and testing files against a [set](https://github.com/jvoisin/php-malware-finder/blob/master/php-malware-finder/php.yar) of [YARA](http://virustotal.github.io/yara/) rules. Yes, it's that simple! -Instead of using an *hash-based* approach, +Instead of using a *hash-based* approach, PMF tries as much as possible to use semantic patterns, to detect things like "a `$_GET` variable is decoded two times, unzipped, and then passed to some dangerous function like `system`". ## Installation -- [Install Yara](https://yara.readthedocs.io/en/stable/gettingstarted.html#compiling-and-installing-yara). -This is also possible via some Linux package managers: - - Debian: `sudo apt-get install yara` - - Red Hat: `yum install yara` (requires the [EPEL repository](https://fedoraproject.org/wiki/EPEL)) - -You can also compile it from source: - -``` -git clone git@github.com:VirusTotal/yara.git -cd yara/ -YACC=bison ./configure -make -``` - -- Download php-malware-finder `git clone https://github.com/jvoisin/php-malware-finder.git` +- Install Go (using your package manager, or [manually](https://go.dev/doc/install)) +- Install libyara >= 4.2 (using your package manager, or [from source](https://yara.readthedocs.io/en/stable/gettingstarted.html)) +- Download php-malware-finder: `git clone https://github.com/jvoisin/php-malware-finder.git` +- Build php-malware-finder: `cd php-malware-finder && make` ## How to use it? ``` $ ./phpmalwarefinder -h -Usage phpmalwarefinder [-cfhtvl] ... - -c Optional path to a rule file - -f Fast mode - -h Show this help message - -t Specify the number of threads to use (8 by default) - -v Verbose mode +Usage: + phpmalwarefinder [OPTIONS] [Target] + +Application Options: + -r, --rules-dir= Rules location (default: /etc/phpmalwarefinder or .) + -a, --show-all Display all matched rules + -f, --fast Enable YARA's fast mode' + -R, --rate-limit= Max. filesystem ops per second, 0 for no limit (default: 0) + -v, --verbose Verbose mode + -w, --workers= Number of workers to spawn for scanning (default: 32) + -L, --long-lines Check long lines + -c, --exclude-common Do not scan files with common extensions + -i, --exclude-imgs Do not scan image files + -x, --exclude-ext= Additional file extensions to exclude + -u, --update Update rules + +Help Options: + -h, --help Show this help message ``` Or if you prefer to use `yara`: diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index e169478..0000000 --- a/debian/changelog +++ /dev/null @@ -1,101 +0,0 @@ -nbs-phpmalwarefinder (0.3.4-1~deb) oldstable; urgency=medium - - * new upstream version : - - update the whitelists - - new rules to prevent bypasses - - readme improvement - - -- jre Mon, 07 Nov 2016 14:26:22 +0100 - -nbs-phpmalwarefinder (0.3.3-1~deb) oldstable; urgency=medium - - * new upstream version : - - add a strrev-based detection - - update the whitelists - - add a new fancy logo - * improve the release process - - -- jvo Mon, 24 Oct 2016 10:02:32 +0200 - -nbs-phpmalwarefinder (0.3.2-1~deb) oldstable; urgency=medium - - * new upstream version : - - whitelists are now split into files, each for one CMS - - a custom whitelist is available for users to add their own - - a mass whitelist helper has been added - * Added the custom whitelist to conffiles to prevent package upgrade from - overwriting users modification. - - -- jre Fri, 29 Jul 2016 09:47:56 +0200 - -nbs-phpmalwarefinder (0.3.1-1~deb) oldstable; urgency=medium - - * new upstream version : - - rules for visbot detection - - now detecting base64 encoded string USER_AGENT - - debian squeeze support dropped - - some false positives fixes - - -- jre Thu, 19 May 2016 15:22:47 +0200 - -nbs-phpmalwarefinder (0.3.0-1~deb) oldstable; urgency=medium - - * rules files refactoring : - - php-malware-finder now comes with asp malware detection - - rules have been split in different files to avoid false positives - - * The -l option allows language specific checks, for now only ASP and PHP - are supported. - * The -u option now allows to update rules without having to upgrade the - package. - - -- jre Thu, 14 Apr 2016 16:04:14 +0200 - -nbs-phpmalwarefinder (0.2.2-1~deb) oldstable; urgency=medium - - * new rules : bad_php.yara to find bad coding practices - * malwares.yara now comes with posix_* functions detection, new hard-coded - strings as well as php:// filter - * The TooShort rule has been improved to reduce FP - - -- jre Mon, 15 Feb 2016 15:48:06 +0100 - -nbs-phpmalwarefinder (0.2.1-1~deb) oldstable; urgency=medium - - * docroot-checker.sh added, helpful for both first and periodic security - scan. - - -- jre Mon, 01 Feb 2016 11:08:08 +0100 - -nbs-phpmalwarefinder (0.2.0-2~deb) oldstable; urgency=medium - - * New detection rules added - - -- sbl Thu, 28 Jan 2016 14:58:45 +0200 - -nbs-phpmalwarefinder (0.2.0-1~deb) oldstable; urgency=medium - - * Now supports whitelist using yara hash function - * New detection rules added (tested against - https://github.com/tennc/webshell malware collection) - - -- jre Fri, 09 Oct 2015 14:58:45 +0200 - -nbs-phpmalwarefinder (0.1.1-1~deb) oldstable; urgency=medium - - * new dependecy on util-linux since the script is using ionice - * postinst script added to create diff folder - - -- jre Tue, 28 Apr 2015 15:07:12 +0200 - -nbs-phpmalwarefinder (0.1.1-1~deb) oldstable; urgency=medium - - * new signature to detect malware in footer and header - - -- jre Tue, 14 Apr 2015 14:40:05 +0000 - -nbs-phpmalwarefinder (0.1) UNRELEASED; urgency=medium - - * Initial release. - - -- jvoisin Tue, 24 Mar 2015 11:10:36 +0100 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index 7ed6ff8..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -5 diff --git a/debian/conffiles b/debian/conffiles deleted file mode 100644 index e69de29..0000000 diff --git a/debian/control b/debian/control deleted file mode 100644 index b50454f..0000000 --- a/debian/control +++ /dev/null @@ -1,14 +0,0 @@ -Source: nbs-phpmalwarefinder -Section: utils -Priority: optional -Maintainer: Security team -Build-Depends: debhelper (>= 8) -Standards-Version: 3.9.5 -Vcs-Git: https://github.com/nbs-system/php-malware-finder -Vcs-Browser: https://github.com/nbs-system/php-malware-finder - -Package: nbs-phpmalwarefinder -Architecture: any -Depends: nbs-yara, wget, nbs-python-yara, python -Description: yara-based php webshell finder - PhpMalwareFinder is a webshell and malware hunter using yara and signatures. diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index 6bec77a..0000000 --- a/debian/copyright +++ /dev/null @@ -1,7 +0,0 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: phpmalwarefinder -Source: https://github.com/nbs-system/php-malware-finder - -Files: * -Copyright 2015 Julien (jvoisin) Voisin -License: GPLv3 diff --git a/debian/files b/debian/files deleted file mode 100644 index 23f95ef..0000000 --- a/debian/files +++ /dev/null @@ -1 +0,0 @@ -nbs-phpmalwarefinder_0.1_amd64.deb utils optional diff --git a/debian/nbs-phpmalwarefinder.dirs b/debian/nbs-phpmalwarefinder.dirs deleted file mode 100644 index 61a8d27..0000000 --- a/debian/nbs-phpmalwarefinder.dirs +++ /dev/null @@ -1 +0,0 @@ -etc/phpmalwarefinder/ \ No newline at end of file diff --git a/debian/nbs-phpmalwarefinder.install b/debian/nbs-phpmalwarefinder.install deleted file mode 100644 index 748222d..0000000 --- a/debian/nbs-phpmalwarefinder.install +++ /dev/null @@ -1,12 +0,0 @@ -whitelists/custom.yar etc/phpmalwarefinder/whitelists -whitelists/drupal.yar etc/phpmalwarefinder/whitelists -whitelists/magento2.yar etc/phpmalwarefinder/whitelists -whitelists/phpmyadmin.yar etc/phpmalwarefinder/whitelists -whitelists/prestashop.yar etc/phpmalwarefinder/whitelists -whitelists/symfony.yar etc/phpmalwarefinder/whitelists -whitelists/wordpress.yar etc/phpmalwarefinder/whitelists -utils/generate_whitelist.py usr/bin/ -utils/mass_whitelist.py usr/bin/ -php.yar etc/phpmalwarefinder -whitelist.yar etc/phpmalwarefinder -phpmalwarefinder usr/bin/ diff --git a/debian/rules b/debian/rules deleted file mode 100755 index bcf500a..0000000 --- a/debian/rules +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/make -f - -BUILDDIR=debian/build - -override_dh_auto_clean: #fuck you debian - -override_dh_auto_build: - -%: - dh $@ - -.PHONY: build diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..39b2f36 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/jvoisin/php-malware-finder + +go 1.17 + +require ( + github.com/hillu/go-yara/v4 v4.2.0 + github.com/jessevdk/go-flags v1.5.0 +) + +require golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..aa0af83 --- /dev/null +++ b/go.sum @@ -0,0 +1,7 @@ +github.com/hillu/go-yara/v4 v4.2.0 h1:C0YycpDYXMlOsN4kbFhvGmfNiaTgpXoLQRS1oUME9ak= +github.com/hillu/go-yara/v4 v4.2.0/go.mod h1:rkb/gSAoO8qcmj+pv6fDZN4tOa3N7R+qqGlEkzT4iys= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/php-malware-finder/phpmalwarefinder b/php-malware-finder/phpmalwarefinder deleted file mode 100755 index a6de360..0000000 --- a/php-malware-finder/phpmalwarefinder +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash - -YARA=$(type -P yara) -CONFIG_PATH='/etc/phpmalwarefinder/php.yar' - -if [ ! -x "$YARA" ] -then - YARA='./yara' - if [ ! -x "$YARA" ] - then - echo 'Unable to find yara in your $PATH, and in the current directory.' - exit 0 - fi -fi - -if [ ! -f "$CONFIG_PATH" ] -then - CONFIG_PATH="$(dirname "$0")/php.yar" -fi - -needle_in_haystack() { - - needle=$(mktemp) - grep -E '(PasswordProtection|Websites|TooShort|NonPrintableChars)' $1 > $needle - if [ ! "$(wc -l "$needle" | awk '{print $1}')" = "0" ]; then - echo "=================================================" - echo "You should take a look at the files listed below:" - cat "$needle" - fi; - rm "$needle" -} - -show_help() { - cat << EOF -Usage ${0##*/} [-cfhtvl] ... - -c Optional path to a rule file - -f Fast mode - -h Show this help message - -t Specify the number of threads to use (8 by default) - -v Verbose mode -EOF -} - -OPTIND=1 -while getopts "c:fht:v" opt; do - case "$opt" in - c) - CONFIG_PATH=${OPTARG} - ;; - f) - OPTS="${OPTS} -f" - ;; - h) - show_help - exit 0 - ;; - t) - OPTS="${OPTS} --threads=${OPTARG}" - ;; - v) - OPTS="${OPTS} -s" - ;; - '?') - show_help - exit 1 - ;; - esac -done -shift "$((OPTIND-1))" - -if [ ! -e "${CONFIG_PATH}" ] -then - echo "The configuration file ${CONFIG_PATH} doesn't exist. Please give me a valid file." - exit 1 -fi - -if [ -z "$@" ] -then - show_help - exit 1 -fi - - -# Include correct yara rule -OPTS="${OPTS} -r ${CONFIG_PATH}" - -# Copy outpout to temporary file -output=$(mktemp) -# delete trailing slash for directories to prevent double slash (issue #40) -target=$(echo "$@" | sed s'#/$##') -# Execute rules -# Using $-interpolation and quotes to support a target with whitespaces -$YARA $OPTS "$target" |tee $output - -needle_in_haystack "$output" -rm "$output" diff --git a/php-malware-finder/phpmalwarefinder.go b/php-malware-finder/phpmalwarefinder.go new file mode 100644 index 0000000..799df60 --- /dev/null +++ b/php-malware-finder/phpmalwarefinder.go @@ -0,0 +1,372 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/hillu/go-yara/v4" + "github.com/jessevdk/go-flags" +) + +const RulesURI = "https://raw.githubusercontent.com/jvoisin/php-malware-finder/master/php-malware-finder/" +const RulesFile = "php.yar" +const DefaultDir = "/etc/phpmalwarefinder" +const ScanMaxDuration = time.Duration(60) +const TooShort = "TooShort" +const TooShortMaxLines = 2 +const TooShortMinChars = 300 +const DangerousMatchWeight = 2 +const DangerousMinScore = 3 +const FileBufferSize = 32 * 1024 // 32KB +const YaraMaxThreads = 32 + +var args struct { // command-line arguments specs using github.com/jessevdk/go-flags + RulesDir string `short:"r" long:"rules-dir" description:"Rules location (default: /etc/phpmalwarefinder or .)"` + ShowAll bool `short:"a" long:"show-all" description:"Display all matched rules"` + Fast bool `short:"f" long:"fast" description:"Enable YARA's fast mode"` + RateLimit int `short:"R" long:"rate-limit" description:"Max. filesystem ops per second, 0 for no limit" default:"0"` + Verbose bool `short:"v" long:"verbose" description:"Verbose mode"` + Workers int `short:"w" long:"workers" description:"Number of workers to spawn for scanning" default:"32"` + LongLines bool `short:"L" long:"long-lines" description:"Check long lines"` + ExcludeCommon bool `short:"c" long:"exclude-common" description:"Do not scan files with common extensions"` + ExcludeImgs bool `short:"i" long:"exclude-imgs" description:"Do not scan image files"` + ExcludedExts []string `short:"x" long:"exclude-ext" description:"Additional file extensions to exclude"` + Update bool `short:"u" long:"update" description:"Update rules"` + Positional struct { + Target string + } `positional-args:"yes"` +} +var scanFlags yara.ScanFlags +var stoppedWorkers int +var lineFeed = []byte{'\n'} +var dangerousMatches = map[string]struct{}{ + "PasswordProtection": {}, + "Websites": {}, + "TooShort": {}, + "NonPrintableChars": {}, +} +var excludedDirs = [...]string{ + "/.git/", "/.hg/", "/.svn/", "/.CVS/", +} +var excludedExts = map[string]struct{}{} +var commonExts = [...]string{ + ".js", ".coffee", ".map", ".min", ".css", ".less", // static files + ".zip", ".rar", ".7z", ".gz", ".bz2", ".xz", ".tar", ".tgz", // archives + ".txt", ".csv", ".json", ".rst", ".md", ".yaml", ".yml", // plain text + ".so", ".dll", ".bin", ".exe", ".bundle", // binaries +} +var imageExts = [...]string{ + ".png", ".jpg", ".jpeg", ".gif", ".svg", ".bmp", ".ico", +} +var scannedFilesCount int + +// handleError is a generic error handler which displays the error message to the user and exits if required. +func handleError(err error, exit bool) { + if err != nil { + log.Println("[ERROR]", err) + if exit { + os.Exit(1) + } + } +} + +// updateRules downloads latest YARA rules from phpmalwarefinder GitHub repository. +// Download location is either `args.RulesDir`, `/etc/phpmalwarefinder`, or the current directory. +func updateRules() { + if args.Verbose { + log.Println("[DEBUG] updating ruleset") + } + + downloadFile := func(uri string) []byte { + resp, err := http.Get(uri) + handleError(err, true) + defer func() { + err := resp.Body.Close() + handleError(err, false) + }() + data, err := ioutil.ReadAll(resp.Body) + handleError(err, true) + return data + } + writeFile := func(dst string, data []byte) { + err := ioutil.WriteFile(dst, data, 0440) + handleError(err, true) + } + + rulesFiles := [...]string{ + RulesFile, + "whitelist.yar", "whitelists/drupal.yar", "whitelists/magento1ce.yar", + "whitelists/magento2.yar", "whitelists/phpmyadmin.yar", "whitelists/prestashop.yar", + "whitelists/symfony.yar", "whitelists/wordpress.yar"} + + // download rules + for _, rule := range rulesFiles { + rulesUri := RulesURI + rule + data := downloadFile(rulesUri) + outPath := path.Join(args.RulesDir, rule) + writeFile(outPath, data) + log.Println("[INFO] updated rule:", rule) + } +} + +// fileStats takes a file path as argument and returns its lines and characters count. +// File reading is done using a 32KB buffer to minimize memory usage. +func fileStats(filepath string) (int, int, error) { + f, err := os.Open(filepath) + handleError(err, true) + defer func() { + err := f.Close() + handleError(err, false) + }() + charCount, lineCount := 0, 0 + buf := make([]byte, FileBufferSize) + for { + chunkSize, err := f.Read(buf) + charCount += chunkSize + lineCount += bytes.Count(buf[:chunkSize], lineFeed) + switch { + case err == io.EOF: + return charCount, lineCount, nil + case err != nil: + return charCount, lineCount, err + } + } +} + +// makeScanner creates a YARA scanner with the appropriate options set. +func makeScanner(rules *yara.Rules) *yara.Scanner { + scanner, err := yara.NewScanner(rules) + handleError(err, true) + scanner.SetFlags(scanFlags) + scanner.SetTimeout(ScanMaxDuration) + return scanner +} + +// processFiles reads file paths from the `targets` channel, scans it, and writes matches to the `results` channel. +// Scanning is done using YARA `rules`, and using `fileStats` if `args.LongLines` is set. +// `ticker` is a `time.Time` object created with `time.Tick` used to throttle file scans to minimize impact on I/O. +func processFiles(rules *yara.Rules, targets <-chan string, results chan<- map[string][]yara.MatchRule, ticker <-chan time.Time) { + scanner := makeScanner(rules) + for target := range targets { + <-ticker + scannedFilesCount++ + result := map[string][]yara.MatchRule{target: {}} + + if args.LongLines { + charCount, lineCount, err := fileStats(target) + handleError(err, false) + if lineCount <= TooShortMaxLines && charCount >= TooShortMinChars { + tooShort := yara.MatchRule{Rule: TooShort} + result[target] = append(result[target], tooShort) + } + } + + var matches yara.MatchRules + err := scanner.SetCallback(&matches).ScanFile(target) + if err != nil { + log.Println("[ERROR]", err) + continue + } + for _, match := range matches { + result[target] = append(result[target], match) + } + results <- result + } + stoppedWorkers++ + if stoppedWorkers == args.Workers { + close(results) + } +} + +// scanDir recursively crawls `dirName`, and writes file paths to the `targets` channel. +// Files sent to `targets` are filtered according to their extensions. +func scanDir(dirName string, targets chan<- string, ticker <-chan time.Time) { + visit := func(pathName string, fileInfo os.FileInfo, err error) error { + <-ticker + if !fileInfo.IsDir() { + for _, dir := range excludedDirs { + if strings.Contains(pathName, dir) { + return nil + } + } + fileExt := filepath.Ext(fileInfo.Name()) + if _, exists := excludedExts[fileExt]; !exists { + targets <- pathName + } + } + return nil + } + err := filepath.Walk(dirName, visit) + handleError(err, false) + close(targets) +} + +func main() { + startTime := time.Now() + _, err := flags.Parse(&args) + handleError(err, true) + + // check rules path + if args.RulesDir == "" { + args.RulesDir = DefaultDir + if _, err := os.Stat(args.RulesDir); os.IsNotExist(err) { + args.RulesDir, _ = os.Getwd() + sigFile := path.Join(args.RulesDir, RulesFile) + if _, err = os.Stat(sigFile); os.IsNotExist(err) { + handleError(fmt.Errorf("no rules in %s or %s", DefaultDir, args.RulesDir), true) + } + } + } + if args.Verbose { + log.Println("[DEBUG] rules directory:", args.RulesDir) + } + + // update rules if required + if args.Update { + updateRules() + os.Exit(0) + } + + // add custom excluded file extensions + if args.ExcludeCommon { + for _, commonExt := range commonExts { + excludedExts[commonExt] = struct{}{} + } + } + if args.ExcludeImgs || args.ExcludeCommon { + for _, imgExt := range imageExts { + excludedExts[imgExt] = struct{}{} + } + } + for _, ext := range args.ExcludedExts { + if string(ext[0]) != "." { + ext = "." + ext + } + excludedExts[ext] = struct{}{} + } + if args.Verbose { + extList := make([]string, len(excludedExts)) + i := 0 + for ext := range excludedExts { + extList[i] = ext[1:] + i++ + } + log.Println("[DEBUG] excluded file extensions:", strings.Join(extList, ",")) + } + + // load YARA rules + rulePath := path.Join(args.RulesDir, RulesFile) + data, _ := ioutil.ReadFile(rulePath) + rules, _ := yara.Compile(string(data), nil) + if args.Verbose { + log.Println("[DEBUG] ruleset loaded:", rulePath) + } + + // set YARA scan flags + if args.Fast { + scanFlags = yara.ScanFlags(yara.ScanFlagsFastMode) + } else { + scanFlags = yara.ScanFlags(0) + } + + // check if requested threads count is not greater than YARA's MAX_THREADS + if args.Workers > YaraMaxThreads { + log.Printf("[WARNING] workers count too high, using %d instead of %d\n", YaraMaxThreads, args.Workers) + args.Workers = YaraMaxThreads + } + + // scan target + if f, err := os.Stat(args.Positional.Target); os.IsNotExist(err) { + handleError(err, true) + } else { + if args.Verbose { + log.Println("[DEBUG] scan workers:", args.Workers) + log.Println("[DEBUG] target:", args.Positional.Target) + } + if f.IsDir() { // parallelized folder scan + // create communication channels + targets := make(chan string) + results := make(chan map[string][]yara.MatchRule) + + // rate limit + var tickerRate time.Duration + if args.RateLimit == 0 { + tickerRate = time.Nanosecond + } else { + tickerRate = time.Second / time.Duration(args.RateLimit) + } + ticker := time.Tick(tickerRate) + if args.Verbose { + log.Println("[DEBUG] delay between fs ops:", tickerRate.String()) + } + + // start consumers and producer workers + for w := 1; w <= args.Workers; w++ { + go processFiles(rules, targets, results, ticker) + } + go scanDir(args.Positional.Target, targets, ticker) + + // read results + matchCount := make(map[string]int) + var keepListing bool + var countedDangerousMatch bool + for result := range results { + for target, matchedSigs := range result { + keepListing = true + matchCount[target] = 0 + countedDangerousMatch = false + for _, sig := range matchedSigs { + matchCount[target] += DangerousMatchWeight + if !countedDangerousMatch { + if _, exists := dangerousMatches[sig.Rule]; exists { + matchCount[target]++ + } + countedDangerousMatch = true + } + if keepListing { + log.Printf("[WARNING] match found: %s (%s)\n", target, sig.Rule) + if !args.ShowAll { + keepListing = false + } + } + } + } + } + for target, count := range matchCount { + if count >= DangerousMinScore { + log.Println("[WARNING] dangerous file found:", target) + } + } + } else { // single file mode + scannedFilesCount++ + var matches yara.MatchRules + scanner := makeScanner(rules) + err := scanner.SetCallback(&matches).ScanFile(args.Positional.Target) + handleError(err, true) + for _, match := range matches { + log.Println("[WARNING] match found:", args.Positional.Target, match.Rule) + if args.Verbose { + for _, matchString := range match.Strings { + log.Printf("[DEBUG] match string for %s: 0x%x:%s: %s\n", args.Positional.Target, matchString.Offset, matchString.Name, matchString.Data) + } + } + if !args.ShowAll { + break + } + } + } + if args.Verbose { + endTime := time.Now() + log.Printf("[DEBUG] scanned %d files in %s\n", scannedFilesCount, endTime.Sub(startTime).String()) + } + } +} diff --git a/php-malware-finder/tests.sh b/php-malware-finder/tests.sh index f53097d..f8c5109 100755 --- a/php-malware-finder/tests.sh +++ b/php-malware-finder/tests.sh @@ -7,7 +7,7 @@ type yara 2>/dev/null 1>&2 || (echo "[-] Please make sure that yara is installed CPT=0 run_test(){ - NB_DETECTED=$(${PMF} -v "$SAMPLES"/"$1" | grep -c "$2" 2>/dev/null) + NB_DETECTED=$(${PMF} -v -a "$SAMPLES"/"$1" 2>&1 | grep -c "$2" 2>/dev/null) if [[ "$NB_DETECTED" != 1 ]]; then echo "[-] $2 was not detected in $1, sorry" -- cgit v1.3