Make application go-install-able and create a docker image

4 files changed, 393 insertions, 0 deletions

diff --git a/utils/generate_whitelist.py b/utils/generate_whitelist.py
new file mode 100755
index 0000000..dabaa21
--- /dev/null
+++ b/utils/generate_whitelist.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+#coding=UTF-8
+
+import fnmatch
+import hashlib
+import os
+import sys
+
+try:
+    import yara
+except ImportError:
+    print('Please install python-yara')
+    sys.exit(1)
+
+if len(sys.argv) != 3:
+    print('Usage: %s name_of_the_rule_and_version folder_to_scan' % sys.argv[0])
+    sys.exit(1)
+
+if not os.path.isdir(sys.argv[2]):
+    print('%s is not a folder !' % sys.argv[2])
+    sys.exit(1)
+
+try:
+    rules = yara.compile(sys.path[0]+'/../php.yar', includes=True, error_on_warning=False)
+except yara.SyntaxError as e:
+    print("Can't compile rules: %s" % e)
+    sys.exit(1)
+
+output_list = list()
+
+for curdir, dirnames, filenames in os.walk(sys.argv[2]):
+    for filename in filenames:
+        fname = os.path.join(curdir, filename)
+        if 0 < os.stat(fname).st_size < 5 * 1024 * 1024:
+            matches = rules.match(fname, fast=True)
+            if matches:
+                with open(fname, 'rb') as f:
+                    digest = hashlib.sha1(f.read()).hexdigest()
+                output_list.append('hash.sha1(0, filesize) == "%s" or // %s' % (digest, fname))
+
+
+if output_list:
+    output_rule = 'import "hash"\n\nrule %s\n{\n\tcondition:\n\t\t/* %s */\n\t\t' % (sys.argv[1].split(' ')[0], sys.argv[1])
+    output_rule += '\n\t\t'.join(output_list)
+    output_rule += '\n\t\tfalse\n}'
+    print(output_rule)
diff --git a/utils/magento1_whitelist.sh b/utils/magento1_whitelist.sh
new file mode 100755
index 0000000..a747f80
--- /dev/null
+++ b/utils/magento1_whitelist.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+# Quit script if something goes wrong
+set  -o errexit  -o nounset  -o pipefail;
+
+SCRIPTDIR="$( dirname "$(readlink -f "$0")" )";
+OUTFILE="${SCRIPTDIR}/../whitelists/magento1ce.yar";
+TMPFILE="${OUTFILE}.new";
+
+# First empty the target whitelist so we can completely generate a new one
+cat <<EOF >"${OUTFILE}";
+private rule Magento1Ce : ECommerce
+{
+        condition:
+                false
+}
+EOF
+
+# Create a temporary directory and make sure it is empty
+GENTEMPDIR="$( mktemp -d --suffix="_gen_whitelist_m1" )";
+
+# Add header to whitelist tempfile
+cat <<EOF | tee "${TMPFILE}";
+private rule Magento1Ce : ECommerce
+{
+        condition:
+EOF
+
+# Fetch tags (releases) from Github repo
+TAGS=$( git ls-remote --tags https://github.com/OpenMage/magento-mirror.git | cut -d '/' -f3 | grep -P '^[\d\.]+$' );
+
+# Foreach tag (release)
+while read -r TAG; do
+    # Download tarball of release
+    wget "https://github.com/OpenMage/magento-mirror/archive/${TAG}.tar.gz" -O "${GENTEMPDIR}/${TAG}.tgz";
+    # Unpack tarball
+    tar -C "${GENTEMPDIR}" -xpzf "${GENTEMPDIR}/${TAG}.tgz";
+    # Add version comment to whitelist tempfile
+    echo "              /* Magento CE ${TAG} */" | tee -a "${TMPFILE}";
+    # Generate whitelist for version, add output to whitelist tempfile
+    ${SCRIPTDIR}/generate_whitelist.py "Magento CE ${TAG}" "${GENTEMPDIR}/magento-mirror-${TAG}" | grep 'hash.sha1' | sed "s|// ${GENTEMPDIR}/magento-mirror-${TAG}/|// |" | tee -a "${TMPFILE}";
+    # Add white line, with indent
+    echo "              " | tee -a "${TMPFILE}";
+done <<< "${TAGS}";
+
+# Add footer to whitelist tempfile
+cat <<EOF | tee -a "${TMPFILE}";
+                false
+}
+EOF
+
+# Copy temporary file to target whitelist while removing duplicate lines except empty ones
+cat "${TMPFILE}" | awk 'match($0,/^\s*$/)||!seen[$0]++' > "${OUTFILE}";
+
+# Clean up
+rm "${TMPFILE}";
+rm -rf "${GENTEMPDIR}";
diff --git a/utils/magento2_whitelist.sh b/utils/magento2_whitelist.sh
new file mode 100755
index 0000000..bb742c8
--- /dev/null
+++ b/utils/magento2_whitelist.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+# Quit script if something goes wrong
+set  -o errexit  -o nounset  -o pipefail;
+
+SCRIPTDIR="$( dirname "$(readlink -f "$0")" )";
+OUTFILE="${SCRIPTDIR}/../whitelists/magento2.yar";
+TMPFILE="${OUTFILE}.new";
+
+# First empty the target whitelist so we can completely generate a new one
+cat <<EOF >"${OUTFILE}";
+private rule Magento2 : ECommerce
+{
+        condition:
+                false
+}
+EOF
+
+# Create a temporary directory and make sure it is empty
+GENTEMPDIR="$( mktemp -d --suffix="_gen_whitelist_m2" )";
+
+# Composer access tokens
+if [ ! -f "${HOME}/.composer/auth.json" ]; then
+    echo -e "\nYou have no '.composer/auth.json' in your home dir. We will create it from a template and open an editor.";
+    echo -e "Press [Enter] to continue. Press Ctrl-C if you wish to leave.";
+    read;
+    mkdir -p "${HOME}/.composer";
+    cat <<EOF >"${HOME}/.composer/auth.json"    
+{
+  "INFO_GITHUB": "==== GET TOKEN: https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/ ====",
+  "github-oauth": {
+    "github.com": "---github-token-goes-here---"
+  },
+  "INFO_MAGENTO": "==== GET TOKEN: https://devdocs.magento.com/guides/v2.0/install-gde/prereq/connect-auth.html ====",
+  "http-basic": {
+    "repo.magento.com": {
+      "username": "---public-key-goes-here---",
+      "password": "---private-key-goes-here---"
+    }
+  }
+}
+EOF
+    editor "${HOME}/.composer/auth.json";
+fi
+
+# Add header to whitelist tempfile
+cat <<EOF | tee "${TMPFILE}";
+private rule Magento2 : ECommerce
+{
+        condition:
+EOF
+
+# Fetch tags (releases) from Github repo
+TAGS=$( git ls-remote --tags https://github.com/magento/magento2.git | cut -d '/' -f3 | grep -P '^[\d\.]+$' | sort -V );
+
+# Foreach tag (release)
+while read -r TAG; do
+    # Download tarball of release
+    wget "https://github.com/magento/magento2/archive/${TAG}.tar.gz" -O "${GENTEMPDIR}/${TAG}.tgz";
+    # Unpack tarball
+    tar -C "${GENTEMPDIR}" -xpzf "${GENTEMPDIR}/${TAG}.tgz";
+    # Run 'composer install' inside unpacked release
+    SOURCEDIR="${GENTEMPDIR}/magento2-${TAG}";
+    composer --working-dir="${SOURCEDIR}" -- install;
+    # Add version comment to whitelist
+    echo "              /* Magento2 ${TAG} */" | tee -a "${TMPFILE}";
+    # Generate whitelist for version, add output to whitelist tempfile
+    ${SCRIPTDIR}/generate_whitelist.py "Magento2 ${TAG}" "${SOURCEDIR}" | grep 'hash.sha1' | sed "s|// ${SOURCEDIR}/|// |" | tee -a "${TMPFILE}";
+    # Add white line, with indent
+    echo "              " | tee -a "${TMPFILE}";
+done <<< "${TAGS}";
+
+# Add footer to whitelist tempfile
+cat <<EOF | tee -a "${TMPFILE}";
+                false
+}
+EOF
+
+# Copy temporary file to target whitelist while removing duplicate lines except empty ones
+cat "${TMPFILE}" | awk 'match($0,/^\s*$/)||!seen[$0]++' > "${OUTFILE}";
+
+# Clean up
+rm "${TMPFILE}";
+rm -rf "${GENTEMPDIR}";
diff --git a/utils/mass_whitelist.py b/utils/mass_whitelist.py
new file mode 100755
index 0000000..868f7b5
--- /dev/null
+++ b/utils/mass_whitelist.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+
+import sys
+import tarfile
+from copy import copy
+from datetime import datetime
+from collections import OrderedDict
+from hashlib import sha1
+from urllib2 import urlopen, HTTPError
+from StringIO import StringIO
+
+import yara
+
+USAGE = """
+USAGE: %(prog)s <NAME> <URL_PATTERN> [<MAJOR> [<MINOR> [<PATCH>]]]
+
+Options:
+    NAME : name of the CMS/whatever being whitelisted
+    URL_PATTERN : download URL with __version__ as a version placeholder
+    MAJOR : minimum and maximum major version to crawl (eg: 1-8, 8)
+    MINOR : minimum and maximum minor version to crawl
+    PATCH : minimum and maximum patch version to crawl
+
+Examples:
+    %(prog)s drupal https://ftp.drupal.org/files/projects/drupal-__version__.tar.gz 9 50
+    %(prog)s drupal https://ftp.drupal.org/files/projects/drupal-__version__.tar.gz 4-9 1-50
+
+    %(prog)s wordpress https://wordpress.org/wordpress-__version__.tar.gz 4 15
+
+    %(prog)s symphony https://github.com/symfony/symfony/archive/v__version__.tar.gz 3 9
+
+    %(prog)s phpmyadmin https://files.phpmyadmin.net/phpMyAdmin/__version__/phpMyAdmin-__version__-all-languages.tar.gz 4 9
+""" % {'prog': sys.argv[0]}
+
+
+class Opts:
+    DEFAULT_MIN = 0
+    DEFAULT_MAX = 99
+    YARA_RULES = yara.compile(sys.path[0]+'/../php.yar', includes=True, error_on_warning=True)
+
+    @classmethod
+    def to_str(cls):
+        values = []
+        for attr in cls.__dict__:
+            if attr.isupper():
+                values.append('%s=%s' % (attr, getattr(cls, attr)))
+        return '<Opts(%s)>' % ' '.join(values)
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def extract_version_arg(index):
+    min_ver, max_ver = (Opts.DEFAULT_MIN, Opts.DEFAULT_MAX)
+    if len(sys.argv) >= (index + 1):
+        if '-' in sys.argv[index]:
+            min_ver, max_ver = map(int, sys.argv[index].split('-'))
+        else:
+            max_ver = int(sys.argv[index])
+    return min_ver, max_ver
+
+
+def generate_whitelist(version):
+    rules = {}
+
+    # download archive
+    dl_failed = False
+    download_url = Opts.URL_PATTERN.replace('__version__', version)
+    download_url_str = Opts.URL_PATTERN.replace('__version__', '\x1b[1;33m%s\x1b[0m' % version)
+    eprint("[+] Downloading %s... " % download_url_str, end='')
+    sys.stdout.flush()
+    try:
+        resp = urlopen(download_url)
+        resp_code = resp.code
+    except HTTPError as err:
+        dl_failed = True
+        resp_code = err.code
+    if dl_failed or (resp_code != 200):
+        eprint("\x1b[1;31mFAILED (%d)\x1b[0m" % resp_code)
+        return None
+    data = StringIO(resp.read())
+    data.seek(0)
+    eprint("\x1b[1;32mOK\x1b[0m")
+
+    # extract archive and check against YARA signatures (in-memory)
+    eprint("[-] Generating whitelist... ", end='')
+    sys.stdout.flush()
+    tar = tarfile.open(mode='r:gz', fileobj=data)
+    for entry in tar.getnames():
+        entry_fd = tar.extractfile(entry)
+        if entry_fd is None:
+            continue
+        entry_data = entry_fd.read()
+        matches = Opts.YARA_RULES.match(data=entry_data, fast=True)
+        if matches:
+            rules['/'.join(entry.split('/')[1:])] = sha1(entry_data).hexdigest()
+    eprint("\x1b[1;32mDONE\x1b[0m")
+
+    return rules
+
+
+# init vars
+whitelists = OrderedDict()
+
+# check args
+if (len(sys.argv) < 3) or (len(sys.argv) > 6):
+    eprint(USAGE)
+    sys.exit(1)
+
+# parse args
+Opts.CMS_NAME = sys.argv[1]
+Opts.URL_PATTERN = sys.argv[2]
+Opts.MIN_MAJOR, Opts.MAX_MAJOR = extract_version_arg(3)
+Opts.MIN_MINOR, Opts.MAX_MINOR = extract_version_arg(4)
+Opts.MIN_PATCH, Opts.MAX_PATCH = extract_version_arg(5)
+
+# loop over possible versions
+for vmajor in range(Opts.MIN_MAJOR, Opts.MAX_MAJOR + 1):
+    # download without vminor and vpatch (but ignore if it doesn't exist)
+    version = "%d" % vmajor
+    rules = generate_whitelist(version)
+    if (rules is not None) and rules:
+        whitelists[version] = rules
+
+    has_mversion = False
+    first_mloop = True
+    for vminor in range(Opts.MIN_MINOR, Opts.MAX_MINOR + 1):
+        # download without vpatch (but ignore if it doesn't exist)
+        version = "%d.%d" % (vmajor, vminor)
+        rules = generate_whitelist(version)
+        if rules is not None:
+            has_mversion = True
+            if rules:
+                whitelists[version] = rules
+        #if (rules is None) and (has_mversion or not first_mloop):
+        #    break
+        first_mloop = False
+
+        has_pversion = False
+        first_ploop = True
+        for vpatch in range(Opts.MIN_PATCH, Opts.MAX_PATCH + 1):
+            version = "%d.%d.%d" % (vmajor, vminor, vpatch)
+            rules = generate_whitelist(version)
+            if rules is not None:
+                has_pversion = True
+                if rules:
+                    whitelists[version] = rules
+            # break loop if download failed and:
+            # - a version has already been found during this loop
+            # - this is the 2nd iteration (if a version wasn't found,
+            #   it means download failed twice)
+            if (rules is None) and (has_pversion or not first_ploop):
+                break
+            first_ploop = False
+
+# remove duplicate entries:
+eprint("[+] Deduplicating detections... ", end='')
+known_files = []
+for version, rules in copy(whitelists.items()):
+    used_rules = 0
+    for filename, digest in rules.items():
+        rtuple = (filename, digest)
+        if rtuple in known_files:
+            del whitelists[version][filename]
+        else:
+            known_files.append(rtuple)
+            used_rules += 1
+    if used_rules == 0:
+        del whitelists[version]
+eprint("\x1b[1;32mDONE\x1b[0m")
+
+eprint("[+] Generating final whitelist... ", end='')
+# build final rule
+prefix = 8 * ' '
+conditions = []
+len_wl = len(whitelists.keys()) - 1
+for index, (version, rules) in enumerate(whitelists.items()):
+    cond_str = '%s/* %s %s */\n' % (prefix, Opts.CMS_NAME.title(), version)
+    len_rules = len(rules.keys()) - 1
+    for inner_index, (filename, digest) in enumerate(rules.items()):
+        if (index == len_wl) and (inner_index == len_rules):  # last loop iteration
+            cond_str += '%shash.sha1(0, filesize) == "%s"    // %s\n' % (prefix, digest, filename)
+        else:
+            cond_str += '%shash.sha1(0, filesize) == "%s" or // %s\n' % (prefix, digest, filename)
+    conditions.append(cond_str)
+eprint("\x1b[1;32mDONE\x1b[0m")
+
+final_rule = """
+import "hash"
+
+private rule %(name)s
+{
+    meta:
+        generated = "%(gendate)s"
+
+    condition:
+%(conditions)s
+}
+""" % {
+    'name': Opts.CMS_NAME.title(),
+    'gendate': datetime.now().isoformat(),
+    'conditions': '\n'.join(conditions)
+}
+print(final_rule)