From 48936efa96ae17295be4e0a71be3294f0ec6aef8 Mon Sep 17 00:00:00 2001 From: Mathieu Deous Date: Mon, 2 May 2022 20:18:23 +0200 Subject: Make application go-install-able and create a docker image --- utils/generate_whitelist.py | 46 ++++++++++ utils/magento1_whitelist.sh | 56 ++++++++++++ utils/magento2_whitelist.sh | 83 ++++++++++++++++++ utils/mass_whitelist.py | 208 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 393 insertions(+) create mode 100755 utils/generate_whitelist.py create mode 100755 utils/magento1_whitelist.sh create mode 100755 utils/magento2_whitelist.sh create mode 100755 utils/mass_whitelist.py (limited to 'utils') diff --git a/utils/generate_whitelist.py b/utils/generate_whitelist.py new file mode 100755 index 0000000..dabaa21 --- /dev/null +++ b/utils/generate_whitelist.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +#coding=UTF-8 + +import fnmatch +import hashlib +import os +import sys + +try: + import yara +except ImportError: + print('Please install python-yara') + sys.exit(1) + +if len(sys.argv) != 3: + print('Usage: %s name_of_the_rule_and_version folder_to_scan' % sys.argv[0]) + sys.exit(1) + +if not os.path.isdir(sys.argv[2]): + print('%s is not a folder !' % sys.argv[2]) + sys.exit(1) + +try: + rules = yara.compile(sys.path[0]+'/../php.yar', includes=True, error_on_warning=False) +except yara.SyntaxError as e: + print("Can't compile rules: %s" % e) + sys.exit(1) + +output_list = list() + +for curdir, dirnames, filenames in os.walk(sys.argv[2]): + for filename in filenames: + fname = os.path.join(curdir, filename) + if 0 < os.stat(fname).st_size < 5 * 1024 * 1024: + matches = rules.match(fname, fast=True) + if matches: + with open(fname, 'rb') as f: + digest = hashlib.sha1(f.read()).hexdigest() + output_list.append('hash.sha1(0, filesize) == "%s" or // %s' % (digest, fname)) + + +if output_list: + output_rule = 'import "hash"\n\nrule %s\n{\n\tcondition:\n\t\t/* %s */\n\t\t' % (sys.argv[1].split(' ')[0], sys.argv[1]) + output_rule += '\n\t\t'.join(output_list) + output_rule += '\n\t\tfalse\n}' + print(output_rule) diff --git a/utils/magento1_whitelist.sh b/utils/magento1_whitelist.sh new file mode 100755 index 0000000..a747f80 --- /dev/null +++ b/utils/magento1_whitelist.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Quit script if something goes wrong +set -o errexit -o nounset -o pipefail; + +SCRIPTDIR="$( dirname "$(readlink -f "$0")" )"; +OUTFILE="${SCRIPTDIR}/../whitelists/magento1ce.yar"; +TMPFILE="${OUTFILE}.new"; + +# First empty the target whitelist so we can completely generate a new one +cat <"${OUTFILE}"; +private rule Magento1Ce : ECommerce +{ + condition: + false +} +EOF + +# Create a temporary directory and make sure it is empty +GENTEMPDIR="$( mktemp -d --suffix="_gen_whitelist_m1" )"; + +# Add header to whitelist tempfile +cat < "${OUTFILE}"; + +# Clean up +rm "${TMPFILE}"; +rm -rf "${GENTEMPDIR}"; diff --git a/utils/magento2_whitelist.sh b/utils/magento2_whitelist.sh new file mode 100755 index 0000000..bb742c8 --- /dev/null +++ b/utils/magento2_whitelist.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Quit script if something goes wrong +set -o errexit -o nounset -o pipefail; + +SCRIPTDIR="$( dirname "$(readlink -f "$0")" )"; +OUTFILE="${SCRIPTDIR}/../whitelists/magento2.yar"; +TMPFILE="${OUTFILE}.new"; + +# First empty the target whitelist so we can completely generate a new one +cat <"${OUTFILE}"; +private rule Magento2 : ECommerce +{ + condition: + false +} +EOF + +# Create a temporary directory and make sure it is empty +GENTEMPDIR="$( mktemp -d --suffix="_gen_whitelist_m2" )"; + +# Composer access tokens +if [ ! -f "${HOME}/.composer/auth.json" ]; then + echo -e "\nYou have no '.composer/auth.json' in your home dir. We will create it from a template and open an editor."; + echo -e "Press [Enter] to continue. Press Ctrl-C if you wish to leave."; + read; + mkdir -p "${HOME}/.composer"; + cat <"${HOME}/.composer/auth.json" +{ + "INFO_GITHUB": "==== GET TOKEN: https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/ ====", + "github-oauth": { + "github.com": "---github-token-goes-here---" + }, + "INFO_MAGENTO": "==== GET TOKEN: https://devdocs.magento.com/guides/v2.0/install-gde/prereq/connect-auth.html ====", + "http-basic": { + "repo.magento.com": { + "username": "---public-key-goes-here---", + "password": "---private-key-goes-here---" + } + } +} +EOF + editor "${HOME}/.composer/auth.json"; +fi + +# Add header to whitelist tempfile +cat < "${OUTFILE}"; + +# Clean up +rm "${TMPFILE}"; +rm -rf "${GENTEMPDIR}"; diff --git a/utils/mass_whitelist.py b/utils/mass_whitelist.py new file mode 100755 index 0000000..868f7b5 --- /dev/null +++ b/utils/mass_whitelist.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import print_function + +import sys +import tarfile +from copy import copy +from datetime import datetime +from collections import OrderedDict +from hashlib import sha1 +from urllib2 import urlopen, HTTPError +from StringIO import StringIO + +import yara + +USAGE = """ +USAGE: %(prog)s [ [ []]] + +Options: + NAME : name of the CMS/whatever being whitelisted + URL_PATTERN : download URL with __version__ as a version placeholder + MAJOR : minimum and maximum major version to crawl (eg: 1-8, 8) + MINOR : minimum and maximum minor version to crawl + PATCH : minimum and maximum patch version to crawl + +Examples: + %(prog)s drupal https://ftp.drupal.org/files/projects/drupal-__version__.tar.gz 9 50 + %(prog)s drupal https://ftp.drupal.org/files/projects/drupal-__version__.tar.gz 4-9 1-50 + + %(prog)s wordpress https://wordpress.org/wordpress-__version__.tar.gz 4 15 + + %(prog)s symphony https://github.com/symfony/symfony/archive/v__version__.tar.gz 3 9 + + %(prog)s phpmyadmin https://files.phpmyadmin.net/phpMyAdmin/__version__/phpMyAdmin-__version__-all-languages.tar.gz 4 9 +""" % {'prog': sys.argv[0]} + + +class Opts: + DEFAULT_MIN = 0 + DEFAULT_MAX = 99 + YARA_RULES = yara.compile(sys.path[0]+'/../php.yar', includes=True, error_on_warning=True) + + @classmethod + def to_str(cls): + values = [] + for attr in cls.__dict__: + if attr.isupper(): + values.append('%s=%s' % (attr, getattr(cls, attr))) + return '' % ' '.join(values) + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def extract_version_arg(index): + min_ver, max_ver = (Opts.DEFAULT_MIN, Opts.DEFAULT_MAX) + if len(sys.argv) >= (index + 1): + if '-' in sys.argv[index]: + min_ver, max_ver = map(int, sys.argv[index].split('-')) + else: + max_ver = int(sys.argv[index]) + return min_ver, max_ver + + +def generate_whitelist(version): + rules = {} + + # download archive + dl_failed = False + download_url = Opts.URL_PATTERN.replace('__version__', version) + download_url_str = Opts.URL_PATTERN.replace('__version__', '\x1b[1;33m%s\x1b[0m' % version) + eprint("[+] Downloading %s... " % download_url_str, end='') + sys.stdout.flush() + try: + resp = urlopen(download_url) + resp_code = resp.code + except HTTPError as err: + dl_failed = True + resp_code = err.code + if dl_failed or (resp_code != 200): + eprint("\x1b[1;31mFAILED (%d)\x1b[0m" % resp_code) + return None + data = StringIO(resp.read()) + data.seek(0) + eprint("\x1b[1;32mOK\x1b[0m") + + # extract archive and check against YARA signatures (in-memory) + eprint("[-] Generating whitelist... ", end='') + sys.stdout.flush() + tar = tarfile.open(mode='r:gz', fileobj=data) + for entry in tar.getnames(): + entry_fd = tar.extractfile(entry) + if entry_fd is None: + continue + entry_data = entry_fd.read() + matches = Opts.YARA_RULES.match(data=entry_data, fast=True) + if matches: + rules['/'.join(entry.split('/')[1:])] = sha1(entry_data).hexdigest() + eprint("\x1b[1;32mDONE\x1b[0m") + + return rules + + +# init vars +whitelists = OrderedDict() + +# check args +if (len(sys.argv) < 3) or (len(sys.argv) > 6): + eprint(USAGE) + sys.exit(1) + +# parse args +Opts.CMS_NAME = sys.argv[1] +Opts.URL_PATTERN = sys.argv[2] +Opts.MIN_MAJOR, Opts.MAX_MAJOR = extract_version_arg(3) +Opts.MIN_MINOR, Opts.MAX_MINOR = extract_version_arg(4) +Opts.MIN_PATCH, Opts.MAX_PATCH = extract_version_arg(5) + +# loop over possible versions +for vmajor in range(Opts.MIN_MAJOR, Opts.MAX_MAJOR + 1): + # download without vminor and vpatch (but ignore if it doesn't exist) + version = "%d" % vmajor + rules = generate_whitelist(version) + if (rules is not None) and rules: + whitelists[version] = rules + + has_mversion = False + first_mloop = True + for vminor in range(Opts.MIN_MINOR, Opts.MAX_MINOR + 1): + # download without vpatch (but ignore if it doesn't exist) + version = "%d.%d" % (vmajor, vminor) + rules = generate_whitelist(version) + if rules is not None: + has_mversion = True + if rules: + whitelists[version] = rules + #if (rules is None) and (has_mversion or not first_mloop): + # break + first_mloop = False + + has_pversion = False + first_ploop = True + for vpatch in range(Opts.MIN_PATCH, Opts.MAX_PATCH + 1): + version = "%d.%d.%d" % (vmajor, vminor, vpatch) + rules = generate_whitelist(version) + if rules is not None: + has_pversion = True + if rules: + whitelists[version] = rules + # break loop if download failed and: + # - a version has already been found during this loop + # - this is the 2nd iteration (if a version wasn't found, + # it means download failed twice) + if (rules is None) and (has_pversion or not first_ploop): + break + first_ploop = False + +# remove duplicate entries: +eprint("[+] Deduplicating detections... ", end='') +known_files = [] +for version, rules in copy(whitelists.items()): + used_rules = 0 + for filename, digest in rules.items(): + rtuple = (filename, digest) + if rtuple in known_files: + del whitelists[version][filename] + else: + known_files.append(rtuple) + used_rules += 1 + if used_rules == 0: + del whitelists[version] +eprint("\x1b[1;32mDONE\x1b[0m") + +eprint("[+] Generating final whitelist... ", end='') +# build final rule +prefix = 8 * ' ' +conditions = [] +len_wl = len(whitelists.keys()) - 1 +for index, (version, rules) in enumerate(whitelists.items()): + cond_str = '%s/* %s %s */\n' % (prefix, Opts.CMS_NAME.title(), version) + len_rules = len(rules.keys()) - 1 + for inner_index, (filename, digest) in enumerate(rules.items()): + if (index == len_wl) and (inner_index == len_rules): # last loop iteration + cond_str += '%shash.sha1(0, filesize) == "%s" // %s\n' % (prefix, digest, filename) + else: + cond_str += '%shash.sha1(0, filesize) == "%s" or // %s\n' % (prefix, digest, filename) + conditions.append(cond_str) +eprint("\x1b[1;32mDONE\x1b[0m") + +final_rule = """ +import "hash" + +private rule %(name)s +{ + meta: + generated = "%(gendate)s" + + condition: +%(conditions)s +} +""" % { + 'name': Opts.CMS_NAME.title(), + 'gendate': datetime.now().isoformat(), + 'conditions': '\n'.join(conditions) +} +print(final_rule) -- cgit v1.3