From 882a1aba2385e1d817ff139e677ad60dec579698 Mon Sep 17 00:00:00 2001
From: Jeroen Vermeulen
Date: Mon, 16 Jul 2018 10:02:01 +0200
Subject: Updated mass_whitelist to output progress to STDERR (#75)

Only the generated rules are printed to STDOUT
Now you can do:
```
./mass_whitelist.py [NAME] [URL_PATTERN] [MAJOR] [MINOR] [PATCH] > ../whitelists/name.yar
```---
 php-malware-finder/utils/mass_whitelist.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/php-malware-finder/utils/mass_whitelist.py b/php-malware-finder/utils/mass_whitelist.py
index c2cf131..868f7b5 100755
--- a/php-malware-finder/utils/mass_whitelist.py
+++ b/php-malware-finder/utils/mass_whitelist.py
@@ -50,6 +50,10 @@ class Opts:
         return '<Opts(%s)>' % ' '.join(values)
 
 
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
 def extract_version_arg(index):
     min_ver, max_ver = (Opts.DEFAULT_MIN, Opts.DEFAULT_MAX)
     if len(sys.argv) >= (index + 1):
@@ -67,7 +71,7 @@ def generate_whitelist(version):
     dl_failed = False
     download_url = Opts.URL_PATTERN.replace('__version__', version)
     download_url_str = Opts.URL_PATTERN.replace('__version__', '\x1b[1;33m%s\x1b[0m' % version)
-    print("[+] Downloading %s... " % download_url_str, end='')
+    eprint("[+] Downloading %s... " % download_url_str, end='')
     sys.stdout.flush()
     try:
         resp = urlopen(download_url)
@@ -76,14 +80,14 @@ def generate_whitelist(version):
         dl_failed = True
         resp_code = err.code
     if dl_failed or (resp_code != 200):
-        print("\x1b[1;31mFAILED (%d)\x1b[0m" % resp_code)
+        eprint("\x1b[1;31mFAILED (%d)\x1b[0m" % resp_code)
         return None
     data = StringIO(resp.read())
     data.seek(0)
-    print("\x1b[1;32mOK\x1b[0m")
+    eprint("\x1b[1;32mOK\x1b[0m")
 
     # extract archive and check against YARA signatures (in-memory)
-    print("[-] Generating whitelist... ", end='')
+    eprint("[-] Generating whitelist... ", end='')
     sys.stdout.flush()
     tar = tarfile.open(mode='r:gz', fileobj=data)
     for entry in tar.getnames():
@@ -94,7 +98,7 @@ def generate_whitelist(version):
         matches = Opts.YARA_RULES.match(data=entry_data, fast=True)
         if matches:
             rules['/'.join(entry.split('/')[1:])] = sha1(entry_data).hexdigest()
-    print("\x1b[1;32mDONE\x1b[0m")
+    eprint("\x1b[1;32mDONE\x1b[0m")
 
     return rules
 
@@ -104,7 +108,7 @@ whitelists = OrderedDict()
 
 # check args
 if (len(sys.argv) < 3) or (len(sys.argv) > 6):
-    print(USAGE)
+    eprint(USAGE)
     sys.exit(1)
 
 # parse args
@@ -132,8 +136,8 @@ for vmajor in range(Opts.MIN_MAJOR, Opts.MAX_MAJOR + 1):
             has_mversion = True
             if rules:
                 whitelists[version] = rules
-        if (rules is None) and (has_mversion or not first_mloop):
-            break
+        #if (rules is None) and (has_mversion or not first_mloop):
+        #    break
         first_mloop = False
 
         has_pversion = False
@@ -154,7 +158,7 @@ for vmajor in range(Opts.MIN_MAJOR, Opts.MAX_MAJOR + 1):
             first_ploop = False
 
 # remove duplicate entries:
-print("[+] Deduplicating detections... ", end='')
+eprint("[+] Deduplicating detections... ", end='')
 known_files = []
 for version, rules in copy(whitelists.items()):
     used_rules = 0
@@ -167,9 +171,9 @@ for version, rules in copy(whitelists.items()):
             used_rules += 1
     if used_rules == 0:
         del whitelists[version]
-print("\x1b[1;32mDONE\x1b[0m")
+eprint("\x1b[1;32mDONE\x1b[0m")
 
-print("[+] Generating final whitelist... ", end='')
+eprint("[+] Generating final whitelist... ", end='')
 # build final rule
 prefix = 8 * ' '
 conditions = []
@@ -183,7 +187,7 @@ for index, (version, rules) in enumerate(whitelists.items()):
         else:
             cond_str += '%shash.sha1(0, filesize) == "%s" or // %s\n' % (prefix, digest, filename)
     conditions.append(cond_str)
-print("\x1b[1;32mDONE\x1b[0m")
+eprint("\x1b[1;32mDONE\x1b[0m")
 
 final_rule = """
 import "hash"
-- 
cgit v1.3