Current File : //lib/python3/dist-packages/py7zr/helpers.py
#!/usr/bin/python -u
#
# p7zr library
#
# Copyright (c) 2019,2020 Hiroshi Miura <miurahr@linux.com>
# Copyright (c) 2004-2015 by Joachim Bauch, mail@joachim-bauch.de
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
#
import array
import ctypes
import io
import mmap
import os
import pathlib
import platform
import sys
import time as _time
import zlib
from datetime import datetime, timedelta, timezone, tzinfo
from typing import BinaryIO, Optional, Union
import _hashlib # type: ignore # noqa
import py7zr.win32compat
from py7zr.properties import READ_BLOCKSIZE
from py7zr import Bad7zFile
from py7zr.win32compat import is_windows_native_python, is_windows_unc_path
# String used at the beginning of relative paths
RELATIVE_PATH_MARKER = "./"
def calculate_crc32(data: bytes, value: int = 0, blocksize: int = 1024 * 1024) -> int:
"""Calculate CRC32 of strings with arbitrary lengths."""
if len(data) <= blocksize:
value = zlib.crc32(data, value)
else:
length = len(data)
pos = blocksize
value = zlib.crc32(data[:pos], value)
while pos < length:
value = zlib.crc32(data[pos:pos + blocksize], value)
pos += blocksize
return value & 0xffffffff
def _calculate_key1(password: bytes, cycles: int, salt: bytes, digest: str) -> bytes:
"""Calculate 7zip AES encryption key. Base implementation. """
if digest not in ('sha256'):
raise ValueError('Unknown digest method for password protection.')
assert cycles <= 0x3f
if cycles == 0x3f:
ba = bytearray(salt + password + bytes(32))
key = bytes(ba[:32]) # type: bytes
else:
rounds = 1 << cycles
m = _hashlib.new(digest)
for round in range(rounds):
m.update(salt + password + round.to_bytes(8, byteorder='little', signed=False))
key = m.digest()[:32]
return key
def _calculate_key2(password: bytes, cycles: int, salt: bytes, digest: str):
"""Calculate 7zip AES encryption key.
It utilize ctypes and memoryview buffer and zero-copy technology on Python."""
if digest not in ('sha256'):
raise ValueError('Unknown digest method for password protection.')
assert cycles <= 0x3f
if cycles == 0x3f:
key = bytes(bytearray(salt + password + bytes(32))[:32]) # type: bytes
else:
rounds = 1 << cycles
m = _hashlib.new(digest)
length = len(salt) + len(password)
class RoundBuf(ctypes.LittleEndianStructure):
_pack_ = 1
_fields_ = [
('saltpassword', ctypes.c_ubyte * length),
('round', ctypes.c_uint64)
]
buf = RoundBuf()
for i, c in enumerate(salt + password):
buf.saltpassword[i] = c
buf.round = 0
mv = memoryview(buf) # type: ignore # noqa
while buf.round < rounds:
m.update(mv)
buf.round += 1
key = m.digest()[:32]
return key
def _calculate_key3(password: bytes, cycles: int, salt: bytes, digest: str) -> bytes:
"""Calculate 7zip AES encryption key.
Concat values in order to reduce number of calls of Hash.update()."""
if digest not in ('sha256'):
raise ValueError('Unknown digest method for password protection.')
assert cycles <= 0x3f
if cycles == 0x3f:
ba = bytearray(salt + password + bytes(32))
key = bytes(ba[:32]) # type: bytes
else:
cat_cycle = 6
if cycles > cat_cycle:
rounds = 1 << cat_cycle
stages = 1 << (cycles - cat_cycle)
else:
rounds = 1 << cycles
stages = 1 << 0
m = _hashlib.new(digest)
saltpassword = salt + password
s = 0 # type: int # (0..stages) * rounds
if platform.python_implementation() == "PyPy":
for _ in range(stages):
m.update(memoryview(b''.join([saltpassword + (s + i).to_bytes(8, byteorder='little', signed=False)
for i in range(rounds)])))
s += rounds
else:
for _ in range(stages):
m.update(b''.join([saltpassword + (s + i).to_bytes(8, byteorder='little', signed=False)
for i in range(rounds)]))
s += rounds
key = m.digest()[:32]
return key
if platform.python_implementation() == "PyPy" or sys.version_info > (3, 6):
calculate_key = _calculate_key3
else:
calculate_key = _calculate_key2 # it is faster when CPython 3.6.x
def filetime_to_dt(ft):
"""Convert Windows NTFS file time into python datetime object."""
EPOCH_AS_FILETIME = 116444736000000000
us = (ft - EPOCH_AS_FILETIME) // 10
return datetime(1970, 1, 1, tzinfo=timezone.utc) + timedelta(microseconds=us)
ZERO = timedelta(0)
HOUR = timedelta(hours=1)
SECOND = timedelta(seconds=1)
# A class capturing the platform's idea of local time.
# (May result in wrong values on historical times in
# timezones where UTC offset and/or the DST rules had
# changed in the past.)
STDOFFSET = timedelta(seconds=-_time.timezone)
if _time.daylight:
DSTOFFSET = timedelta(seconds=-_time.altzone)
else:
DSTOFFSET = STDOFFSET
DSTDIFF = DSTOFFSET - STDOFFSET
class LocalTimezone(tzinfo):
def fromutc(self, dt):
assert dt.tzinfo is self
stamp = (dt - datetime(1970, 1, 1, tzinfo=self)) // SECOND
args = _time.localtime(stamp)[:6]
dst_diff = DSTDIFF // SECOND
# Detect fold
fold = (args == _time.localtime(stamp - dst_diff))
return datetime(*args, microsecond=dt.microsecond, tzinfo=self)
def utcoffset(self, dt):
if self._isdst(dt):
return DSTOFFSET
else:
return STDOFFSET
def dst(self, dt):
if self._isdst(dt):
return DSTDIFF
else:
return ZERO
def tzname(self, dt):
return _time.tzname[self._isdst(dt)]
def _isdst(self, dt):
tt = (dt.year, dt.month, dt.day,
dt.hour, dt.minute, dt.second,
dt.weekday(), 0, 0)
stamp = _time.mktime(tt)
tt = _time.localtime(stamp)
return tt.tm_isdst > 0
Local = LocalTimezone()
TIMESTAMP_ADJUST = -11644473600
class UTC(tzinfo):
"""UTC"""
def utcoffset(self, dt):
return ZERO
def tzname(self, dt):
return "UTC"
def dst(self, dt):
return ZERO
def _call__(self):
return self
class ArchiveTimestamp(int):
"""Windows FILETIME timestamp."""
def __repr__(self):
return '%s(%d)' % (type(self).__name__, self)
def __index__(self):
return self.__int__()
def totimestamp(self) -> float:
"""Convert 7z FILETIME to Python timestamp."""
# FILETIME is 100-nanosecond intervals since 1601/01/01 (UTC)
return (self / 10000000.0) + TIMESTAMP_ADJUST
def as_datetime(self):
"""Convert FILETIME to Python datetime object."""
return datetime.fromtimestamp(self.totimestamp(), UTC())
@staticmethod
def from_datetime(val):
return ArchiveTimestamp((val - TIMESTAMP_ADJUST) * 10000000.0)
def islink(path):
"""
Cross-platform islink implementation.
Support Windows NT symbolic links and reparse points.
"""
is_symlink = os.path.islink(str(path))
if sys.version_info >= (3, 8) or sys.platform != "win32" or sys.getwindowsversion()[0] < 6:
return is_symlink
# special check for directory junctions which py38 does.
if is_symlink:
if py7zr.win32compat.is_reparse_point(path):
is_symlink = False
return is_symlink
def readlink(path: Union[str, pathlib.Path], *, dir_fd=None) -> Union[str, pathlib.Path]:
"""
Cross-platform compat implementation of os.readlink and Path.readlink().
Support Windows NT symbolic links and reparse points.
When called with path argument as pathlike(str), return result as a pathlike(str).
When called with Path object, return also Path object.
When called with path argument as bytes, return result as a bytes.
"""
is_path_pathlib = isinstance(path, pathlib.Path)
if sys.version_info >= (3, 9):
if is_path_pathlib and dir_fd is None:
return path.readlink()
else:
return os.readlink(path, dir_fd=dir_fd)
elif sys.version_info >= (3, 8) or sys.platform != "win32":
res = os.readlink(path, dir_fd=dir_fd)
# Hack to handle a wrong type of results
if isinstance(res, bytes):
res = os.fsdecode(res)
if is_path_pathlib:
return pathlib.Path(res)
else:
return res
elif not os.path.exists(str(path)):
raise OSError(22, 'Invalid argument', path)
return py7zr.win32compat.readlink(path)
class MemIO:
"""pathlib.Path-like IO class to write memory(io.Bytes)"""
def __init__(self, buf: BinaryIO):
self._buf = buf
def write(self, data: bytes) -> int:
return self._buf.write(data)
def read(self, length: Optional[int] = None) -> bytes:
if length is not None:
return self._buf.read(length)
else:
return self._buf.read()
def close(self) -> None:
self._buf.seek(0)
def flush(self) -> None:
pass
def seek(self, position: int) -> None:
self._buf.seek(position)
def open(self, mode=None):
return self
@property
def parent(self):
return self
def mkdir(self, parents=None, exist_ok=False):
return None
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
class NullIO:
"""pathlib.Path-like IO class of /dev/null"""
def __init__(self):
pass
def write(self, data):
return len(data)
def read(self, length=None):
if length is not None:
return bytes(length)
else:
return b''
def close(self):
pass
def flush(self):
pass
def open(self, mode=None):
return self
@property
def parent(self):
return self
def mkdir(self):
return None
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
class BufferOverflow(Exception):
pass
class Buffer:
def __init__(self, size: int = 16):
self._buf = bytearray(size)
self._buflen = 0
self.view = memoryview(self._buf[0:0])
def add(self, data: Union[bytes, bytearray, memoryview]):
length = len(data)
self._buf[self._buflen:] = data
self._buflen += length
self.view = memoryview(self._buf[0:self._buflen])
def reset(self) -> None:
self._buflen = 0
self.view = memoryview(self._buf[0:0])
def set(self, data: Union[bytes, bytearray, memoryview]) -> None:
length = len(data)
self._buf[0:] = data
self._buflen = length
self.view = memoryview(self._buf[0:length])
def get(self) -> bytearray:
val = self._buf[:self._buflen]
self.reset()
return val
def __len__(self) -> int:
return self._buflen
def __bytes__(self):
return bytes(self._buf[0:self._buflen])
class BufferedRW(io.BufferedIOBase):
def __init__(self):
self._buf = bytearray()
def writable(self):
return True
def write(self, b: Union[bytes, bytearray, memoryview, array.array, mmap.mmap]):
if isinstance(b, mmap.mmap):
size = b.size()
current = b.tell()
if size - current > READ_BLOCKSIZE:
self._buf += b.read(READ_BLOCKSIZE)
elif size - current > 0:
self._buf += b.read(size - current)
elif isinstance(b, array.array):
self._buf += b.tobytes()
else:
self._buf += b
def readable(self):
return True
def read(self, size: Optional[int] = -1):
if size is None or size < 0:
length: int = len(self._buf)
else:
length = size
result = bytes(self._buf[:length])
self._buf[:] = self._buf[length:]
return result
def readinto(self, b) -> int:
length = min(len(self._buf), len(b))
b[:] = self._buf[:length]
self._buf[:] = self._buf[length:]
return length
def __len__(self):
return len(self._buf)
def remove_relative_path_marker(path: str) -> str:
"""
Removes './' from the beginning of a path-like string
"""
processed_path = path
if path.startswith(RELATIVE_PATH_MARKER):
processed_path = path[len(RELATIVE_PATH_MARKER) :]
return processed_path
def canonical_path(target: pathlib.PurePath) -> pathlib.PurePath:
"""Return a canonical path of target argument."""
stack: List[str] = []
for p in target.parts:
if p != ".." or len(stack) == 0:
stack.append(p)
continue
# treat '..'
if stack[-1] == "..":
stack.append(p) # '../' + '../' -> '../../'
elif stack[-1] == "/":
pass # '/' + '../' -> '/'
else:
stack.pop() # 'foo/boo/' + '..' -> 'foo/'
return pathlib.PurePath(*stack)
def is_relative_to(my: pathlib.PurePath, *other) -> bool:
"""Return True when path is relative to other path, otherwise False."""
try:
my.relative_to(*other)
except ValueError:
return False
return True
def get_sanitized_output_path(fname: str, path: Optional[pathlib.Path]) -> pathlib.Path:
"""
check f.filename has invalid directory traversals
When condition is not satisfied, raise Bad7zFile
"""
if path is None:
target_path = canonical_path(pathlib.Path.cwd().joinpath(fname))
if is_relative_to(target_path, pathlib.Path.cwd()):
return pathlib.Path(remove_relative_path_marker(fname))
else:
outfile = canonical_path(path.joinpath(remove_relative_path_marker(fname)))
if is_relative_to(outfile, path):
return pathlib.Path(outfile)
raise Bad7zFile(f"Specified path is bad: {fname}")
def check_archive_path(arcname: str) -> bool:
"""
Check arcname argument is valid for archive.
It should not be absolute, if so it returns False.
It should not be evil traversal attack path.
Otherwise, returns True.
"""
if pathlib.PurePath(arcname).is_absolute():
return False
# test against dummy parent path
if sys.platform == "win32":
path = pathlib.Path("C:/foo/boo/fuga/hoge/a90sufoiasj09/dafj08sajfa/")
else:
path = pathlib.Path("/foo/boo/fuga/hoge/a90sufoiasj09/dafj08sajfa/")
return is_path_valid(path.joinpath(arcname), path)
def get_sanitized_output_path(fname: str, path: Optional[pathlib.Path]) -> pathlib.Path:
"""
check f.filename has invalid directory traversals
do following but is_relative_to introduced in py 3.9,
so I replaced it with relative_to. when condition is not satisfied, raise ValueError
if not pathlib.Path(...).joinpath(remove_relative_path_marker(outname)).is_relative_to(...):
raise Bad7zFile
"""
if path is None:
try:
pathlib.Path(os.getcwd()).joinpath(fname).resolve().relative_to(os.getcwd())
outfile = pathlib.Path(remove_relative_path_marker(fname))
except ValueError:
raise Bad7zFile(f"Specified path is bad: {fname}")
else:
try:
outfile = path.joinpath(remove_relative_path_marker(fname))
outfile.resolve().relative_to(path)
except ValueError:
raise Bad7zFile(f"Specified path is bad: {fname}")
return outfile
def check_archive_path(arcname: str) -> bool:
path = pathlib.Path("/foo/boo/fuga/hoge/a90sufoiasj09/dafj08sajfa/") # dummy path
return is_target_path_valid(path, path.joinpath(arcname))
def is_target_path_valid(path: pathlib.Path, target: pathlib.Path) -> bool:
try:
if path.is_absolute():
target.resolve().relative_to(path)
else:
target.resolve().relative_to(pathlib.Path(os.getcwd()).joinpath(path))
except ValueError:
return False
return True
def check_win32_file_namespace(pathname: pathlib.Path) -> pathlib.Path:
# When python on Windows and not python on Cygwin,
# Add win32 file namespace to exceed Microsoft Windows
# path length limitation to 260 bytes
# ref.
# https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
# In editions of Windows before Windows 10 version 1607,
# the maximum length for a path is MAX_PATH, which is defined as
# 260 characters. In later versions of Windows, changing a registry key
# or select option when python installation is required to remove the limit.
if is_windows_native_python() and pathname.is_absolute() and not is_windows_unc_path(pathname):
pathname = pathlib.WindowsPath("\\\\?\\" + str(pathname))
return pathname
Mini Shell