Current File : //backups/router/usr/local/opnsense/scripts/suricata/lib/downloader.py
"""
Copyright (c) 2015-2018 Ad Schellevis <ad@opnsense.org>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------------
rule downloader module (may need to be extended in the future, this version only processes http(s))
"""
import syslog
import tarfile
import gzip
import zipfile
import tempfile
import requests
import json
import hashlib
import os
import re
class Downloader(object):
def __init__(self, target_dir):
self._target_dir = target_dir
self._download_cache = dict()
@staticmethod
def _unpack(src, source_filename, filename=None):
""" unpack data if archived
:param src: handle to temp file
:param source_filename: original source filename
:param filename: filename to extract
:return: string
"""
src.seek(0)
unpack_type = None
if source_filename.endswith('.tar.gz') or source_filename.endswith('.tgz'):
unpack_type = 'tar'
elif source_filename.endswith('.gz'):
unpack_type = 'gz'
elif source_filename.endswith('.zip'):
unpack_type = 'zip'
if unpack_type is not None:
rule_content = list()
# handle compression types
if unpack_type == 'tar':
tf = tarfile.open(fileobj=src)
for tf_file in tf.getmembers():
# extract partial or all (*.rules) from archive
if filename is not None and tf_file.name == filename:
rule_content.append(tf.extractfile(tf_file).read())
elif filename is None and tf_file.isfile() and tf_file.name.lower().endswith('.rules'):
rule_content.append(tf.extractfile(tf_file).read())
elif unpack_type == 'gz':
gf = gzip.GzipFile(mode='r', fileobj=src)
rule_content.append(gf.read())
elif unpack_type == 'zip':
with zipfile.ZipFile(src, mode='r', compression=zipfile.ZIP_DEFLATED) as zf:
for item in zf.infolist():
if filename is not None and item.filename == filename:
rule_content.append(zf.open(item).read())
elif filename is None and item.file_size > 0 and item.filename.lower().endswith('.rules'):
rule_content.append(zf.open(item).read())
return '\n'.join(x.decode() for x in rule_content)
else:
return src.read().decode()
def fetch(self, url, auth=None, headers=None):
""" Fetch file from remote location and save to temp, return filehandle pointed to start of temp file.
Results are cached, which prevents downloading the same archive twice for example.
:param url: download url
:param auth: authentication
:param headers: headers to send
:return: dict {'handle': filehandle, 'filename': filename}
"""
if str(url).split(':')[0].lower() in ('http', 'https'):
frm_url = url.replace('//', '/').replace(':/', '://')
# stream to temp file
if frm_url not in self._download_cache:
req_opts = {
'url': frm_url,
'stream': True
}
if auth is not None:
req_opts['auth'] = auth
if headers is not None:
req_opts['headers'] = headers
try:
req = requests.get(**req_opts)
except requests.exceptions.RequestException as e:
syslog.syslog(syslog.LOG_ERR, 'download failed for %s (%s)' % (url, e))
return None
if 'content-disposition' not in req.headers \
or req.headers['content-disposition'].find('filename=') == -1:
filename = url.strip().lower().split('?')[0]
else:
filename = re.findall('filename=(.+)', req.headers['content-disposition'])[0].strip('"')
if req.status_code == 200:
req.raw.decode_content = True
src = tempfile.NamedTemporaryFile('wb+', 10240)
while True:
data = req.raw.read(10240)
if not data:
break
else:
src.write(data)
self._download_cache[frm_url] = {'handle': src, 'filename': filename, 'cached': False}
else:
syslog.syslog(syslog.LOG_ERR, 'download failed for %s (http_code: %d)' % (url, req.status_code))
else:
self._download_cache[frm_url]['cached'] = True
else:
syslog.syslog(syslog.LOG_ERR, 'unsupported download type for %s' % (url))
if frm_url in self._download_cache:
self._download_cache[frm_url]['handle'].seek(0)
return self._download_cache[frm_url]
else:
return None
def fetch_version_hash(self, check_url, auth=None, headers=None):
""" Calculate a hash value using the download settings and a predefined version url (check_url).
:param check_url: download url, version identifier
:param auth: authentication
:param headers: headers to send
:return: None or hash
"""
if check_url is not None:
# when no check url provided, assume different
if self.is_supported(check_url):
version_fetch = self.fetch(url=check_url, auth=auth, headers=headers)
if version_fetch:
version_response = version_fetch['handle'].read().decode()
hash_value = [json.dumps(auth), json.dumps(headers), version_response]
if not version_fetch['cached']:
syslog.syslog(syslog.LOG_NOTICE, 'version response for %s : %s' % (check_url, version_response))
return hashlib.md5(('\n'.join(hash_value)).encode()).hexdigest()
return None
def installed_file_hash(self, filename):
""" Fetch file version hash from header
:param filename: target filename
:return: None or hash
"""
target_filename = '%s/%s' % (self._target_dir, filename)
if os.path.isfile(target_filename):
with open(target_filename, 'r') as f_in:
line = f_in.readline()
if line.find("#@opnsense_download_hash:") == 0:
return line.split(':')[1].strip()
return None
def download(self, url, url_filename, filename, auth=None, headers=None, version=None):
""" download ruleset file
:param url: download url
:param url_filename: if provided the filename within the (packet) resource
:param filename: target filename
:param auth: authentication
:param headers: headers to send
:param version: version hash
"""
frm_url = url.replace('//', '/').replace(':/', '://')
fetch_result = self.fetch(url=url, auth=auth, headers=headers)
if fetch_result is not None:
try:
target_filename = '%s/%s' % (self._target_dir, filename)
if version:
save_data = "#@opnsense_download_hash:%s\n" % version
else:
save_data = ""
save_data += self._unpack(
src=fetch_result['handle'], source_filename=fetch_result['filename'],
filename=url_filename
)
open(target_filename, 'w', buffering=10240).write(save_data)
except IOError:
syslog.syslog(syslog.LOG_ERR, 'cannot write to %s' % target_filename)
return None
except UnicodeDecodeError:
syslog.syslog(syslog.LOG_ERR, 'unable to read %s from %s (decode error)' % (
target_filename, fetch_result['filename']
))
return None
if not fetch_result['cached']:
syslog.syslog(syslog.LOG_NOTICE, 'download completed for %s' % frm_url)
@staticmethod
def is_supported(url: str) -> bool:
""" check if protocol is supported
:param url: uri to request resource from
:return: bool
"""
return str(url).split(':')[0].lower() in ('http', 'https')
Mini Shell