From 4b349d339d3531edda150d3df3d08e9a1aec1f1d Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Tue, 27 Mar 2018 21:11:48 -0300 Subject: [PATCH 01/74] Resolve pyx requirement for Python 2 and 3 --- requirements-docs.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 074a8a1..d459929 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -2,6 +2,7 @@ Sphinx==1.6.5 ipykernel nbconvert nbsphinx -pyx==0.12.1 +pyx==0.12.1; python_version < '3.2' +pyx==0.14.1; python_version >= '3.2' ipython<6.0 m2r==0.1.13 \ No newline at end of file From 475b47551f677ccaf285a589c1b2a38b689b815e Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Tue, 27 Mar 2018 21:12:14 -0300 Subject: [PATCH 02/74] Enable Python 3 build on AppVeyor and Travis --- .appveyor.yml | 10 ++++++++++ .travis.yml | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 0e36e88..781899d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -16,6 +16,16 @@ environment: ARCH: "64" PLAT_NAME: "win-amd64" + - PYTHON: "C:\\Python27" + PYTHON_VERSION: "3.6.x" + ARCH: "32" + PLAT_NAME: "win32" + + - PYTHON: "C:\\Python27-x64" + PYTHON_VERSION: "3.6.x" + ARCH: "64" + PLAT_NAME: "win-amd64" + # Install requirements install: # Set path diff --git a/.travis.yml b/.travis.yml index da62728..c99879d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,31 @@ matrix: env: - CXX=g++ CC=gcc + - os: linux + python: 3.3 + env: + - CXX=clang++ CC=clang + + - os: linux + python: 3.4 + env: + - CXX=clang++ CC=clang + + - os: linux + python: 3.5 + env: + - CXX=clang++ CC=clang + + - os: linux + python: 3.6 + env: + - CXX=clang++ CC=clang + + - os: linux + python: 3.6 + env: + - CXX=g++ CC=gcc + - os: osx language: generic @@ -36,12 +61,14 @@ before_script: brew install https://raw.githubusercontent.com/secdev/scapy/master/.travis/pylibpcap.rb; brew install pandoc; fi + # Determine the pip vesion to use + - PIP=`which pip || (python --version 2>&1 | grep -q 'Python 2' && which pip2) || (python --version 2>&1 | grep -q 'Python 3' && which pip3)` install: - - pip2 install . - - pip2 install six - - pip2 install -r requirements-docs.txt - - pip2 install -r requirements-optional.txt + - $PIP install . + - $PIP install six + - $PIP install -r requirements-docs.txt + - $PIP install -r requirements-optional.txt script: - python setup.py test From a9c11e0b6bb6b62ed3c454968fe4a3bda2d1117b Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Thu, 5 Jul 2018 14:20:54 -0300 Subject: [PATCH 03/74] Fix PIP version --- .travis.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 767a3a3..c30d917 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,11 +17,6 @@ matrix: env: - CXX=g++ CC=gcc - - os: linux - python: 3.3 - env: - - CXX=clang++ CC=clang - - os: linux python: 3.4 env: @@ -61,14 +56,15 @@ before_script: brew install https://raw.githubusercontent.com/secdev/scapy/master/.travis/pylibpcap.rb; brew install openssl pandoc; fi - # Determine the pip vesion to use - - PIP=`which pip || (python --version 2>&1 | grep -q 'Python 2' && which pip2) || (python --version 2>&1 | grep -q 'Python 3' && which pip3)` install: - - $PIP install . - - $PIP install flake8 six - - $PIP install -r requirements-docs.txt - - $PIP install -r requirements-optional.txt + # Determine the pip vesion to use + - PIP=`which pip || (python --version 2>&1 | grep -q 'Python 2' && which pip2) || (python --version 2>&1 | grep -q 'Python 3' && which pip3)` + - ${PIP} --version + - ${PIP} install . + - ${PIP} install flake8 six + - ${PIP} install -r requirements-docs.txt + - ${PIP} install -r requirements-optional.txt # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide From 3ff02732a858f22c71ac8b67e94aaf4ee798c389 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 5 Jul 2018 10:00:01 +0300 Subject: [PATCH 04/74] Make C extension module init Python 2/3 compatible --- pysapcompress/pysapcompress.cpp | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/pysapcompress/pysapcompress.cpp b/pysapcompress/pysapcompress.cpp index 2819587..03b7e79 100644 --- a/pysapcompress/pysapcompress.cpp +++ b/pysapcompress/pysapcompress.cpp @@ -366,7 +366,7 @@ static char pysapcompress_compress_doc[] = "Compress a buffer using SAP's compre ":param str in: input buffer to compress\n\n" ":param int algorithm: algorithm to use\n\n" ":return: tuple with return code, output length and output buffer\n" - ":rtype: tuple of int, int, string\n\n" + ":rtype: tuple of int, int, bytes\n\n" ":raises CompressError: if an error occurred during compression\n"; static PyObject * @@ -409,7 +409,7 @@ static char pysapcompress_decompress_doc[] = "Decompress a buffer using SAP's co ":param str in: input buffer to decompress\n" ":param int out_length: length of the output to decompress\n" ":return: tuple of return code, output length and output buffer\n" - ":rtype: tuple of int, int, string\n\n" + ":rtype: tuple of int, int, bytes\n\n" ":raises DecompressError: if an error occurred during decompression\n"; static PyObject * @@ -452,12 +452,31 @@ static PyMethodDef pysapcompressMethods[] = { static char pysapcompress_module_doc[] = "Library implementing SAP's LZH and LZC compression algorithms."; /* Module initialization */ -PyMODINIT_FUNC -initpysapcompress(void) +/* Python 2 and 3 compatiblitily shenanigans, from http://python3porting.com/cextensions.html */ +#if PY_MAJOR_VERSION >= 3 + #define MOD_ERROR_VAL NULL + #define MOD_SUCCESS_VAL(val) val + #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) + #define MOD_DEF(ob, name, doc, methods) \ + static struct PyModuleDef moduledef = { \ + PyModuleDef_HEAD_INIT, name, doc, -1, methods, }; \ + ob = PyModule_Create(&moduledef); +#else + #define MOD_ERROR_VAL + #define MOD_SUCCESS_VAL(val) + #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) + #define MOD_DEF(ob, name, doc, methods) \ + ob = Py_InitModule3(name, methods, doc); +#endif + +MOD_INIT(pysapcompress) { PyObject *module = NULL; /* Create the module and define the methods */ - module = Py_InitModule3("pysapcompress", pysapcompressMethods, pysapcompress_module_doc); + MOD_DEF(module, "pysapcompress", pysapcompress_module_doc, pysapcompressMethods) + + if (module == NULL) + return MOD_ERROR_VAL; /* Add the algorithm constants */ PyModule_AddIntConstant(module, "ALG_LZC", ALG_LZC); @@ -470,4 +489,5 @@ initpysapcompress(void) decompression_exception = PyErr_NewException(decompression_exception_name, NULL, NULL); PyModule_AddObject(module, decompression_exception_short, decompression_exception); + return MOD_SUCCESS_VAL(module); } From 4c34b550769be23ffe4834c826dd4cadedb1f713 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 5 Jul 2018 13:12:11 +0300 Subject: [PATCH 05/74] Make pysapcompress C extension Python 2/3 compatible --- pysapcompress/pysapcompress.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pysapcompress/pysapcompress.cpp b/pysapcompress/pysapcompress.cpp index 03b7e79..41491dd 100644 --- a/pysapcompress/pysapcompress.cpp +++ b/pysapcompress/pysapcompress.cpp @@ -34,6 +34,13 @@ #include "hpa107cslzh.h" #include "hpa105CsObjInt.h" +/* compress and decompress take and return bytes on Py3, unfortunately there's no way to enforce it on Py2 */ +#if PY_MAJOR_VERSION >= 3 + #define BYTES_FORMAT "y#" +#else + #define BYTES_FORMAT "s#" +#endif + /* Memory allocation factor constant */ #define MEMORY_ALLOC_FACTOR 10 @@ -382,7 +389,7 @@ pysapcompress_compress(PyObject *self, PyObject *args, PyObject *keywds) static char* kwlist[] = {kwin, kwalgorithm, NULL}; /* Parse the parameters. We are also interested in the length of the input buffer. */ - if (!PyArg_ParseTupleAndKeywords(args, keywds, "s#|i", kwlist, &in, &in_length, &algorithm)) + if (!PyArg_ParseTupleAndKeywords(args, keywds, BYTES_FORMAT"|i", kwlist, &in, &in_length, &algorithm)) return (NULL); /* Call the compression function */ @@ -400,7 +407,7 @@ pysapcompress_compress(PyObject *self, PyObject *args, PyObject *keywds) } /* It no error was raised, return the compressed buffer and the length */ - return (Py_BuildValue("iis#", status, out_length, out, out_length)); + return (Py_BuildValue("ii"BYTES_FORMAT, status, out_length, out, out_length)); } @@ -420,7 +427,7 @@ pysapcompress_decompress(PyObject *self, PyObject *args) int status = 0, in_length = 0, out_length = 0; /* Parse the parameters. We are also interested in the length of the input buffer. */ - if (!PyArg_ParseTuple(args, "s#i", &in, &in_length, &out_length)) + if (!PyArg_ParseTuple(args, BYTES_FORMAT"i", &in, &in_length, &out_length)) return (NULL); /* Call the compression function */ @@ -436,7 +443,7 @@ pysapcompress_decompress(PyObject *self, PyObject *args) return (PyErr_Format(decompression_exception, "Decompression error (%s)", error_string(status))); } /* It no error was raised, return the uncompressed buffer and the length */ - return (Py_BuildValue("iis#", status, out_length, out, out_length)); + return (Py_BuildValue("ii"BYTES_FORMAT, status, out_length, out, out_length)); } From f8ca38da815d87971f118727f097b5bf9c735a9e Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 5 Jul 2018 13:21:38 +0300 Subject: [PATCH 06/74] Make pysapcompress unit tests Python 2/3 compatible Basically only change is to make sure we handle data as bytes instead of strings. --- tests/pysapcompress_test.py | 19 ++++++++++--------- tests/utils.py | 8 ++++---- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/pysapcompress_test.py b/tests/pysapcompress_test.py index 1aa297b..6c57345 100755 --- a/tests/pysapcompress_test.py +++ b/tests/pysapcompress_test.py @@ -18,6 +18,7 @@ # ============== # Standard imports +from __future__ import unicode_literals import unittest # Custom imports from tests.utils import read_data_file @@ -25,29 +26,29 @@ class PySAPCompressTest(unittest.TestCase): - test_string_plain = "TEST" * 70 - test_string_compr_lzc = '\x18\x01\x00\x00\x11\x1f\x9d\x8dT\x8aL\xa1\x12p`A\x82\x02\x11\x1aLx\xb0!\xc3\x87\x0b#*\x9c\xe8PbE\x8a\x101Z\xccx\xb1#\xc7\x8f\x1bCj\x1c\xe9QdI\x92 Q\x9aLy\xf2 ' - test_string_compr_lzh = '\x18\x01\x00\x00\x12\x1f\x9d\x02]\x88kpH\xc8(\xc6\xc0\x00\x00' + test_string_plain = b"TEST" * 70 + test_string_compr_lzc = b'\x18\x01\x00\x00\x11\x1f\x9d\x8dT\x8aL\xa1\x12p`A\x82\x02\x11\x1aLx\xb0!\xc3\x87\x0b#*\x9c\xe8PbE\x8a\x101Z\xccx\xb1#\xc7\x8f\x1bCj\x1c\xe9QdI\x92 Q\x9aLy\xf2 ' + test_string_compr_lzh = b'\x18\x01\x00\x00\x12\x1f\x9d\x02]\x88kpH\xc8(\xc6\xc0\x00\x00' def test_import(self): """Test import of the pysapcompress library""" try: import pysapcompress # @UnusedImport except ImportError as e: - self.Fail(str(e)) + self.fail(str(e)) def test_compress_input(self): """Test compress function input""" from pysapcompress import compress, CompressError - self.assertRaisesRegexp(CompressError, "invalid input length", compress, "") - self.assertRaisesRegexp(CompressError, "unknown algorithm", compress, "TestString", algorithm=999) + self.assertRaisesRegexp(CompressError, "invalid input length", compress, b"") + self.assertRaisesRegexp(CompressError, "unknown algorithm", compress, b"TestString", algorithm=999) def test_decompress_input(self): """Test decompress function input""" from pysapcompress import decompress, DecompressError - self.assertRaisesRegexp(DecompressError, "invalid input length", decompress, "", 1) - self.assertRaisesRegexp(DecompressError, "input not compressed", decompress, "AAAAAAAA", 1) - self.assertRaisesRegexp(DecompressError, "unknown algorithm", decompress, "\x0f\x00\x00\x00\xff\x1f\x9d\x00\x00\x00\x00", 1) + self.assertRaisesRegexp(DecompressError, "invalid input length", decompress, b"", 1) + self.assertRaisesRegexp(DecompressError, "input not compressed", decompress, b"AAAAAAAA", 1) + self.assertRaisesRegexp(DecompressError, "unknown algorithm", decompress, b"\x0f\x00\x00\x00\xff\x1f\x9d\x00\x00\x00\x00", 1) def test_lzc(self): """Test compression and decompression using LZC algorithm""" diff --git a/tests/utils.py b/tests/utils.py index dab4bb9..983fd34 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -19,19 +19,19 @@ # Standard imports from binascii import unhexlify -from os.path import join as join, dirname +from os import path def data_filename(filename): - return join(dirname(__file__), 'data', filename) + return path.join(path.dirname(__file__), 'data', filename) def read_data_file(filename, unhex=True): filename = data_filename(filename) - with open(filename, 'r') as f: + with open(filename, 'rb') as f: data = f.read() - data = data.replace('\n', ' ').replace(' ', '') + data = data.replace(b'\n', b' ').replace(b' ', b'') if unhex: data = unhexlify(data) From 02e95f2c3b19d16a16bf19ee4e0aab61903e6e04 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 5 Jul 2018 13:23:10 +0300 Subject: [PATCH 07/74] Use six to get rid of deprecation warning on Py3 --- requirements.txt | 3 ++- tests/pysapcompress_test.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index ca1a042..f3d2dce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -scapy==2.4.0 \ No newline at end of file +six==1.11.0 +scapy==2.4.0 diff --git a/tests/pysapcompress_test.py b/tests/pysapcompress_test.py index 6c57345..7e6df09 100755 --- a/tests/pysapcompress_test.py +++ b/tests/pysapcompress_test.py @@ -19,6 +19,7 @@ # Standard imports from __future__ import unicode_literals +import six import unittest # Custom imports from tests.utils import read_data_file @@ -35,20 +36,20 @@ def test_import(self): try: import pysapcompress # @UnusedImport except ImportError as e: - self.fail(str(e)) + self.fail(six.text_type(e)) def test_compress_input(self): """Test compress function input""" from pysapcompress import compress, CompressError - self.assertRaisesRegexp(CompressError, "invalid input length", compress, b"") - self.assertRaisesRegexp(CompressError, "unknown algorithm", compress, b"TestString", algorithm=999) + six.assertRaisesRegex(self, CompressError, "invalid input length", compress, b"") + six.assertRaisesRegex(self, CompressError, "unknown algorithm", compress, b"TestString", algorithm=999) def test_decompress_input(self): """Test decompress function input""" from pysapcompress import decompress, DecompressError - self.assertRaisesRegexp(DecompressError, "invalid input length", decompress, b"", 1) - self.assertRaisesRegexp(DecompressError, "input not compressed", decompress, b"AAAAAAAA", 1) - self.assertRaisesRegexp(DecompressError, "unknown algorithm", decompress, b"\x0f\x00\x00\x00\xff\x1f\x9d\x00\x00\x00\x00", 1) + six.assertRaisesRegex(self, DecompressError, "invalid input length", decompress, b"", 1) + six.assertRaisesRegex(self, DecompressError, "input not compressed", decompress, b"AAAAAAAA", 1) + six.assertRaisesRegex(self, DecompressError, "unknown algorithm", decompress, b"\x0f\x00\x00\x00\xff\x1f\x9d\x00\x00\x00\x00", 1) def test_lzc(self): """Test compression and decompression using LZC algorithm""" @@ -155,7 +156,7 @@ def test_invalid_write(self): test_case = read_data_file('invalid_write_testcase.data', False) - self.assertRaisesRegexp(DecompressError, "stack overflow in decomp", decompress, test_case, 6716) + six.assertRaisesRegex(self, DecompressError, "stack overflow in decomp", decompress, test_case, 6716) def test_invalid_read(self): """Test invalid read vulnerability in LZH code (CVE-2015-2278)""" From 3b7d43bded8f05ea9313012f126bb68386c96ce0 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 5 Jul 2018 13:47:26 +0300 Subject: [PATCH 08/74] Add Python requirements and update CI accordingly Scapy 2.4.0 only works with Python 2.7 and Python 3.4 and beyond, so restrict compatible Python versions for pysap as well. --- .travis.yml | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c30d917..5a8b163 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,7 +62,7 @@ install: - PIP=`which pip || (python --version 2>&1 | grep -q 'Python 2' && which pip2) || (python --version 2>&1 | grep -q 'Python 3' && which pip3)` - ${PIP} --version - ${PIP} install . - - ${PIP} install flake8 six + - ${PIP} install flake8 - ${PIP} install -r requirements-docs.txt - ${PIP} install -r requirements-optional.txt # stop the build if there are Python syntax errors or undefined names diff --git a/setup.py b/setup.py index fc43426..d7ba080 100755 --- a/setup.py +++ b/setup.py @@ -96,6 +96,7 @@ def run(self): url=pysap.__url__, download_url=pysap.__url__, license=pysap.__license__, + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4', classifiers=['Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'Intended Audience :: Information Technology', From 815961246091c878dd944cbfd78259eb38cfaa9d Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Fri, 6 Jul 2018 15:58:46 +0300 Subject: [PATCH 09/74] Use os.path methods from path to retain context --- tests/sapcar_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/sapcar_test.py b/tests/sapcar_test.py index 33ee186..2c42cfa 100755 --- a/tests/sapcar_test.py +++ b/tests/sapcar_test.py @@ -19,8 +19,7 @@ # Standard imports import unittest -from os import unlink -from os.path import basename, exists +from os import unlink, path # External imports # Custom imports from tests.utils import data_filename @@ -44,7 +43,7 @@ def setUp(self): def tearDown(self): for filename in [self.test_filename, self.test_archive_file]: - if exists(filename): + if path.exists(filename): unlink(filename) def check_sapcar_archive(self, filename, version): @@ -53,7 +52,7 @@ def check_sapcar_archive(self, filename, version): with open(data_filename(filename), "rb") as fd: sapcar_archive = SAPCARArchive(fd, mode="r") - self.assertEqual(filename, basename(sapcar_archive.filename)) + self.assertEqual(filename, path.basename(sapcar_archive.filename)) self.assertEqual(version, sapcar_archive.version) self.assertEqual(1, len(sapcar_archive.files)) self.assertEqual(1, len(sapcar_archive.files_names)) From 138888bea1cfa23a41c1b7efbc00a152dd549bcf Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Fri, 6 Jul 2018 15:21:07 +0300 Subject: [PATCH 10/74] Move filemode code under Python version check _filemode_table and filemode function are copied from Python 3 stat, so just import filemode from stat if running Python 3. --- pysap/SAPCAR.py | 90 +++++++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/pysap/SAPCAR.py b/pysap/SAPCAR.py index c39a0e4..06b4aa8 100644 --- a/pysap/SAPCAR.py +++ b/pysap/SAPCAR.py @@ -18,6 +18,7 @@ # ============== # Standard imports +import six import stat from zlib import crc32 from struct import pack @@ -31,55 +32,56 @@ ConditionalField, LESignedIntField, StrField, LELongField) # Custom imports from pysap.utils.fields import (PacketNoPadded, StrNullFixedLenField, PacketListStopField) -from pysapcompress import (decompress, compress, ALG_LZH, CompressError, - DecompressError) - - -# Filemode code obtained from Python 3 stat.py -_filemode_table = ( - ((stat.S_IFLNK, "l"), - (stat.S_IFREG, "-"), - (stat.S_IFBLK, "b"), - (stat.S_IFDIR, "d"), - (stat.S_IFCHR, "c"), - (stat.S_IFIFO, "p")), - - ((stat.S_IRUSR, "r"),), - ((stat.S_IWUSR, "w"),), - ((stat.S_IXUSR|stat.S_ISUID, "s"), - (stat.S_ISUID, "S"), - (stat.S_IXUSR, "x")), - - ((stat.S_IRGRP, "r"),), - ((stat.S_IWGRP, "w"),), - ((stat.S_IXGRP|stat.S_ISGID, "s"), - (stat.S_ISGID, "S"), - (stat.S_IXGRP, "x")), - - ((stat.S_IROTH, "r"),), - ((stat.S_IWOTH, "w"),), - ((stat.S_IXOTH|stat.S_ISVTX, "t"), - (stat.S_ISVTX, "T"), - (stat.S_IXOTH, "x")) -) +from pysapcompress import (decompress, compress, ALG_LZH, CompressError, DecompressError) + + +if six.PY2: + # Filemode code obtained from Python 3 stat.py + _filemode_table = ( + ((stat.S_IFLNK, "l"), + (stat.S_IFREG, "-"), + (stat.S_IFBLK, "b"), + (stat.S_IFDIR, "d"), + (stat.S_IFCHR, "c"), + (stat.S_IFIFO, "p")), + + ((stat.S_IRUSR, "r"),), + ((stat.S_IWUSR, "w"),), + ((stat.S_IXUSR|stat.S_ISUID, "s"), + (stat.S_ISUID, "S"), + (stat.S_IXUSR, "x")), + + ((stat.S_IRGRP, "r"),), + ((stat.S_IWGRP, "w"),), + ((stat.S_IXGRP|stat.S_ISGID, "s"), + (stat.S_ISGID, "S"), + (stat.S_IXGRP, "x")), + + ((stat.S_IROTH, "r"),), + ((stat.S_IWOTH, "w"),), + ((stat.S_IXOTH|stat.S_ISVTX, "t"), + (stat.S_ISVTX, "T"), + (stat.S_IXOTH, "x")) + ) + + def filemode(mode): + """Convert a file's mode to a string of the form '-rwxrwxrwx'.""" + perm = [] + for table in _filemode_table: + for bit, char in table: + if mode & bit == bit: + perm.append(char) + break + else: + perm.append("-") + return "".join(perm) +else: + from stat import filemode SIZE_FOUR_GB = 0xffffffff + 1 -def filemode(mode): - """Convert a file's mode to a string of the form '-rwxrwxrwx'.""" - perm = [] - for table in _filemode_table: - for bit, char in table: - if mode & bit == bit: - perm.append(char) - break - else: - perm.append("-") - return "".join(perm) - - class SAPCARInvalidFileException(Exception): """Exception to denote an invalid SAP CAR file""" From 2bb2de5bd3d94814cc63a305d4e8ca2e0afe4c35 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Fri, 6 Jul 2018 15:26:12 +0300 Subject: [PATCH 11/74] Add function to convert bytes and unicode to unicode .decode() can be a real PITA as it works fine for unicode strings in legacy Python but barfs on Python 3. Instead of putting if six.PY2 checks all over the repo, I created a helper function that works on pretty much every string on both Python 2 and 3. Also created six module under utils, where other custom general purpose Python 2/3 compatibility code can reside. --- pysap/utils/six.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 pysap/utils/six.py diff --git a/pysap/utils/six.py b/pysap/utils/six.py new file mode 100644 index 0000000..65d47fa --- /dev/null +++ b/pysap/utils/six.py @@ -0,0 +1,39 @@ +# =========== +# pysap - Python library for crafting SAP's network protocols packets +# +# Copyright (C) 2012-2018 by Martin Gallo, Core Security +# +# The library was designed and developed by Martin Gallo from +# Core Security's CoreLabs team. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# ============== + +# All custom general purpose Python 2/3 compatibility should go here + +from __future__ import absolute_import + +import six + + +def unicode(string): + """ + Convert given string to unicode string + :param string: String to convert + :type string: bytes | str | unicode + :return: six.text_type + """ + string_type = type(string) + if string_type == six.binary_type: + return string.decode() + elif string_type == six.text_type: + return string + raise ValueError("Expected bytes or str, got {}".format(string_type)) From c998f0597b58bafed3f46f2ca13ad490448ea578 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Fri, 6 Jul 2018 15:43:28 +0300 Subject: [PATCH 12/74] Make distinction between bytes and unicode strings Also updated docstrings to use constants from six, dunno if IDEs actually handle those but at least they're unambiguous regardless of Python version. --- pysap/SAPCAR.py | 119 ++++++++++++++++++++++--------------------- tests/sapcar_test.py | 3 +- 2 files changed, 63 insertions(+), 59 deletions(-) diff --git a/pysap/SAPCAR.py b/pysap/SAPCAR.py index 06b4aa8..f249fe9 100644 --- a/pysap/SAPCAR.py +++ b/pysap/SAPCAR.py @@ -18,6 +18,7 @@ # ============== # Standard imports +from __future__ import unicode_literals import six import stat from zlib import crc32 @@ -32,6 +33,7 @@ ConditionalField, LESignedIntField, StrField, LELongField) # Custom imports from pysap.utils.fields import (PacketNoPadded, StrNullFixedLenField, PacketListStopField) +from pysap.utils.six import unicode from pysapcompress import (decompress, compress, ALG_LZH, CompressError, DecompressError) @@ -100,8 +102,8 @@ class SAPCARCompressedBlobFormat(PacketNoPadded): fields_desc = [ LEIntField("compressed_length", None), LEIntField("uncompress_length", None), - ByteEnumField("algorithm", 0x12, {0x12: "LZH", 0x10: "LZC"}), - StrFixedLenField("magic_bytes", "\x1f\x9d", 2), + ByteEnumField("algorithm", 0x12, {0x12: b"LZH", 0x10: b"LZC"}), + StrFixedLenField("magic_bytes", b"\x1f\x9d", 2), ByteField("special", 2), ConditionalField(StrField("blob", None, remain=4), lambda x: x.compressed_length <= 8), ConditionalField(StrFixedLenField("blob", None, length_from=lambda x: x.compressed_length - 8), @@ -109,16 +111,16 @@ class SAPCARCompressedBlobFormat(PacketNoPadded): ] -SAPCAR_BLOCK_TYPE_COMPRESSED_LAST = "ED" +SAPCAR_BLOCK_TYPE_COMPRESSED_LAST = b"ED" """SAP CAR compressed end of data block""" -SAPCAR_BLOCK_TYPE_COMPRESSED = "DA" +SAPCAR_BLOCK_TYPE_COMPRESSED = b"DA" """SAP CAR compressed block""" -SAPCAR_BLOCK_TYPE_UNCOMPRESSED_LAST = "UE" +SAPCAR_BLOCK_TYPE_UNCOMPRESSED_LAST = b"UE" """SAP CAR uncompressed end of data block""" -SAPCAR_BLOCK_TYPE_UNCOMPRESSED = "UD" +SAPCAR_BLOCK_TYPE_UNCOMPRESSED = b"UD" """SAP CAR uncompressed block""" @@ -150,22 +152,22 @@ def sapcar_is_last_block(packet): return packet.type in [SAPCAR_BLOCK_TYPE_COMPRESSED_LAST, SAPCAR_BLOCK_TYPE_UNCOMPRESSED_LAST] -SAPCAR_TYPE_FILE = "RG" +SAPCAR_TYPE_FILE = b"RG" """SAP CAR regular file string""" -SAPCAR_TYPE_DIR = "DR" +SAPCAR_TYPE_DIR = b"DR" """SAP CAR directory string""" -SAPCAR_TYPE_SHORTCUT = "SC" +SAPCAR_TYPE_SHORTCUT = b"SC" """SAP CAR Windows short cut string""" -SAPCAR_TYPE_LINK = "LK" +SAPCAR_TYPE_LINK = b"LK" """SAP CAR Unix soft link string""" -SAPCAR_TYPE_AS400 = "SV" +SAPCAR_TYPE_AS400 = b"SV" """SAP CAR AS400 save file string""" - +# Version strings are unicode instead of byte strings in order to avoid constant .decode() and .encode() calls SAPCAR_VERSION_200 = "2.00" """SAP CAR file format version 2.00 string""" @@ -234,7 +236,7 @@ def extract(self, fd): if self.file_length == 0: return 0 - compressed = "" + compressed = b"" checksum = 0 exp_length = None @@ -247,7 +249,7 @@ def extract(self, fd): # Store compressed block types for later decompression elif block.type in [SAPCAR_BLOCK_TYPE_COMPRESSED, SAPCAR_BLOCK_TYPE_COMPRESSED_LAST]: # Add compressed block to a buffer, skipping the first 4 bytes of each block (uncompressed length) - compressed += str(block.compressed)[4:] + compressed += six.binary_type(block.compressed)[4:] # If the expected length wasn't already set, do it if not exp_length: exp_length = block.compressed.uncompress_length @@ -259,7 +261,7 @@ def extract(self, fd): checksum = block.checksum # If there was at least one compressed block that set the expected length, decompress it if exp_length: - (_, block_length, block_buffer) = decompress(str(compressed), exp_length) + (_, block_length, block_buffer) = decompress(six.binary_type(compressed), exp_length) if block_length != exp_length or not block_buffer: raise DecompressError("Error decompressing block") fd.write(block_buffer) @@ -279,10 +281,10 @@ class SAPCARArchiveFilev201Format(SAPCARArchiveFilev200Format): is_filename_null_terminated = True -SAPCAR_HEADER_MAGIC_STRING_STANDARD = "CAR\x20" +SAPCAR_HEADER_MAGIC_STRING_STANDARD = b"CAR\x20" """SAP CAR archive header magic string standard""" -SAPCAR_HEADER_MAGIC_STRING_BACKUP = "CAR\x00" +SAPCAR_HEADER_MAGIC_STRING_BACKUP = b"CAR\x00" """SAP CAR archive header magic string backup file""" @@ -303,10 +305,11 @@ class SAPCARArchiveFormat(Packet): fields_desc = [ StrFixedLenField("magic_string", SAPCAR_HEADER_MAGIC_STRING_STANDARD, 4), StrFixedLenField("version", SAPCAR_VERSION_201, 4), + # Scapy automatically encodes unicode strings, so need to decode packet version in lambda ConditionalField(PacketListField("files0", None, SAPCARArchiveFilev200Format), - lambda x: x.version == SAPCAR_VERSION_200), + lambda x: x.version.decode() == SAPCAR_VERSION_200), ConditionalField(PacketListField("files1", None, SAPCARArchiveFilev201Format), - lambda x: x.version == SAPCAR_VERSION_201), + lambda x: x.version.decode() == SAPCAR_VERSION_201), ] @@ -348,34 +351,34 @@ def version(self): """The version of the file. :return: version of the file - :rtype: string + :rtype: six.text_type """ - return self._file_format.version + return unicode(self._file_format.version) @property def type(self): """The type of the file. :return: type of the file - :rtype: string + :rtype: six.text_type """ - return self._file_format.type + return unicode(self._file_format.type) @property def filename(self): """The name of the file. :return: name of the file - :rtype: string + :rtype: six.text_type """ - return self._file_format.filename + return unicode(self._file_format.filename) @filename.setter def filename(self, filename): """Sets the name of the file. :param filename: the name of the file - :type filename: string + :type filename: six.text_type """ self._file_format.filename = filename self._file_format.filename_length = len(filename) @@ -405,9 +408,9 @@ def permissions(self): """The permissions of the file. :return: permissions in human-readable format - :rtype: string + :rtype: six.text_type """ - return filemode(self._file_format.perm_mode) + return unicode(filemode(self._file_format.perm_mode)) @permissions.setter def permissions(self, perm_mode): @@ -432,9 +435,9 @@ def timestamp(self): """The timestamp of the file. :return: timestamp in human-readable format - :rtype: string + :rtype: six.text_type """ - return datetime.utcfromtimestamp(self._file_format.timestamp).strftime('%d %b %Y %H:%M') + return unicode(datetime.utcfromtimestamp(self._file_format.timestamp).strftime('%d %b %Y %H:%M')) @timestamp.setter def timestamp(self, timestamp): @@ -496,7 +499,7 @@ def calculate_checksum(data): """Calculates the CRC32 checksum of a given data string. :param data: data to calculate the checksum over - :type data: str + :type data: six.binary_type :return: the CRC32 checksum :rtype: int @@ -509,13 +512,13 @@ def from_file(cls, filename, version=SAPCAR_VERSION_201, archive_filename=None): local file system. :param filename: filename to build the file format object from - :type filename: string + :type filename: six.text_type :param version: version of the file to construct - :type version: string + :type version: six.text_type :param archive_filename: filename to use inside the archive file - :type archive_filename: string + :type archive_filename: six.text_type :raise ValueError: if the version requested is invalid """ @@ -568,7 +571,7 @@ def from_archive_file(cls, archive_file, version=SAPCAR_VERSION_201): :type archive_file: L{SAPCARArchiveFile} :param version: version of the file to construct - :type version: string + :type version: six.text_type :raise ValueError: if the version requested is invalid """ @@ -589,7 +592,7 @@ def from_archive_file(cls, archive_file, version=SAPCAR_VERSION_201): for block in archive_file._file_format.blocks: new_block = SAPCARCompressedBlockFormat() new_block.type = block.type - new_block.compressed = SAPCARCompressedBlobFormat(str(block.compressed)) + new_block.compressed = SAPCARCompressedBlobFormat(six.binary_type(block.compressed)) new_block.checksum = block.checksum new_archive_file._file_format.blocks.append(new_block) @@ -653,13 +656,13 @@ def __init__(self, fil, mode="rb+", version=SAPCAR_VERSION_201): """Opens an archive file and allow access to it. :param fil: filename or file descriptor to open - :type fil: string or file + :type fil: six.text_type or file :param mode: mode to open the file - :type mode: string + :type mode: six.text_type :param version: archive file version to use when creating - :type version: string + :type version: six.text_type """ # Ensure version is withing supported versions @@ -674,7 +677,7 @@ def __init__(self, fil, mode="rb+", version=SAPCAR_VERSION_201): if "b" not in mode: mode += "b" - if isinstance(fil, (basestring, unicode)): + if isinstance(fil, six.text_type): self.filename = fil self.fd = open(fil, mode) else: @@ -691,13 +694,13 @@ def __init__(self, fil, mode="rb+", version=SAPCAR_VERSION_201): def files(self): """The list of file objects inside this archive file. - :return: list of file objects - :rtype: L{dict} of L{SAPCARArchiveFile} + :return: dictionary of file objects + :rtype: dict(six.text_type, SAPCARArchiveFile) """ fils = {} if self._files: for fil in self._files: - fils[fil.filename] = SAPCARArchiveFile(fil) + fils[fil.filename.decode()] = SAPCARArchiveFile(fil) return fils @property @@ -705,7 +708,7 @@ def files_names(self): """The list of file names inside this archive file. :return: list of file names - :rtype: L{list} of L{string} + :rtype: list(six.text_type) """ return self.files.keys() @@ -714,9 +717,9 @@ def version(self): """The version of the archive file. :return: version - :rtype: string + :rtype: six.text_type """ - return self._sapcar.version + return unicode(self._sapcar.version) @version.setter def version(self, version): @@ -724,7 +727,7 @@ def version(self, version): converts the archive file. :param version: version to set - :type version: string + :type version: six.text_type """ if version not in sapcar_archive_file_versions: raise ValueError("Invalid version") @@ -749,7 +752,7 @@ def read(self): self._sapcar = SAPCARArchiveFormat(self.fd.read()) if self._sapcar.magic_string not in [SAPCAR_HEADER_MAGIC_STRING_STANDARD, SAPCAR_HEADER_MAGIC_STRING_BACKUP]: raise Exception("Invalid or unsupported magic string in file") - if self._sapcar.version not in sapcar_archive_file_versions: + if self._sapcar.version.decode() not in sapcar_archive_file_versions: raise Exception("Invalid or unsupported version in file") @property @@ -779,29 +782,29 @@ def write(self): """Writes the SAP CAR archive file to the file descriptor. """ self.fd.seek(0) - self.fd.write(str(self._sapcar)) + self.fd.write(six.binary_type(self._sapcar)) self.fd.flush() def write_as(self, filename=None): """Writes the SAP CAR archive file to another file. :param filename: name of the file to write to - :type filename: string + :type filename: six.text_type """ if not filename: self.write() else: - with open(filename, "w") as fd: - fd.write(str(self._sapcar)) + with open(filename, "wb") as fd: + fd.write(six.binary_type(self._sapcar)) def add_file(self, filename, archive_filename=None): """Adds a new file to the SAP CAR archive file. :param filename: name of the file to add - :type filename: string + :type filename: six.text_type :param archive_filename: name of the file to use in the archive - :type archive_filename: string + :type archive_filename: six.text_type """ fil = SAPCARArchiveFile.from_file(filename, self.version, archive_filename) self._files.append(fil._file_format) @@ -811,7 +814,7 @@ def open(self, filename): inside the SAP CAR archive. :param filename: name of the file to open - :type filename: string + :type filename: six.text_type :return: a file-like object that can be used to access the decompressed file. :rtype: file @@ -829,8 +832,8 @@ def raw(self): """Returns the raw data of the archive file. :return: raw data - :rtype: string + :rtype: six.binary_type """ if self._sapcar: - return str(self._sapcar) - return "" + return six.binary_type(self._sapcar) + return b"" diff --git a/tests/sapcar_test.py b/tests/sapcar_test.py index 2c42cfa..2ca6d4e 100755 --- a/tests/sapcar_test.py +++ b/tests/sapcar_test.py @@ -18,6 +18,7 @@ # ============== # Standard imports +from __future__ import unicode_literals import unittest from os import unlink, path # External imports @@ -35,7 +36,7 @@ class PySAPCARTest(unittest.TestCase): test_timestamp = "01 Dec 2015 22:48" test_perm_mode = 33204 test_permissions = "-rw-rw-r--" - test_string = "The quick brown fox jumps over the lazy dog" + test_string = b"The quick brown fox jumps over the lazy dog" def setUp(self): with open(self.test_filename, "wb") as fd: From 091429b957ab25b5fbab3a7d77c94c657bbb2846 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Fri, 6 Jul 2018 15:49:11 +0300 Subject: [PATCH 13/74] Use BytesIO instead of cStringIO We're handling binary data after all. --- pysap/SAPCAR.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysap/SAPCAR.py b/pysap/SAPCAR.py index f249fe9..02a8ebd 100644 --- a/pysap/SAPCAR.py +++ b/pysap/SAPCAR.py @@ -25,7 +25,7 @@ from struct import pack from datetime import datetime from os import stat as os_stat -from cStringIO import StringIO +from io import BytesIO # External imports from scapy.packet import Packet from scapy.fields import (ByteField, ByteEnumField, LEIntField, FieldLenField, @@ -618,7 +618,7 @@ def open(self, enforce_checksum=False): raise Exception("Invalid file type") # Extract the file to a file-like object - out_file = StringIO() + out_file = BytesIO() checksum = self._file_format.extract(out_file) out_file.seek(0) From 29dba1b2cf770517dca3ee269b6293ea6fe056db Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Fri, 6 Jul 2018 15:56:45 +0300 Subject: [PATCH 14/74] Rename variable to prevent shadowing module name --- pysap/SAPCAR.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pysap/SAPCAR.py b/pysap/SAPCAR.py index 02a8ebd..b8455f1 100644 --- a/pysap/SAPCAR.py +++ b/pysap/SAPCAR.py @@ -524,7 +524,7 @@ def from_file(cls, filename, version=SAPCAR_VERSION_201, archive_filename=None): """ # Read the file properties and its content - stat = os_stat(filename) + fil_stat = os_stat(filename) with open(filename, "rb") as fd: data = fd.read() @@ -547,9 +547,9 @@ def from_file(cls, filename, version=SAPCAR_VERSION_201, archive_filename=None): # Build the object and fill the fields archive_file = cls() archive_file._file_format = ff() - archive_file._file_format.perm_mode = stat.st_mode - archive_file._file_format.timestamp = stat.st_atime - archive_file._file_format.file_length = stat.st_size + archive_file._file_format.perm_mode = fil_stat.st_mode + archive_file._file_format.timestamp = fil_stat.st_atime + archive_file._file_format.file_length = fil_stat.st_size archive_file._file_format.filename = archive_filename archive_file._file_format.filename_length = len(archive_filename) if archive_file._file_format.version == SAPCAR_VERSION_201: From 25430371b187b403c06870d9c51f9c626d2814a5 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Fri, 6 Jul 2018 15:57:18 +0300 Subject: [PATCH 15/74] Convert dict_keys object to list in Python 3 --- pysap/SAPCAR.py | 5 ++++- tests/sapcar_test.py | 11 +++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pysap/SAPCAR.py b/pysap/SAPCAR.py index b8455f1..0ab2222 100644 --- a/pysap/SAPCAR.py +++ b/pysap/SAPCAR.py @@ -710,7 +710,10 @@ def files_names(self): :return: list of file names :rtype: list(six.text_type) """ - return self.files.keys() + if six.PY2: + return self.files.keys() + # In Python 3, dict.keys() returns fancy new dict_keys object that needs type conversion + return list(self.files.keys()) @property def version(self): diff --git a/tests/sapcar_test.py b/tests/sapcar_test.py index 2ca6d4e..f5b2e55 100755 --- a/tests/sapcar_test.py +++ b/tests/sapcar_test.py @@ -19,6 +19,7 @@ # Standard imports from __future__ import unicode_literals +import six import unittest from os import unlink, path # External imports @@ -58,7 +59,10 @@ def check_sapcar_archive(self, filename, version): self.assertEqual(1, len(sapcar_archive.files)) self.assertEqual(1, len(sapcar_archive.files_names)) self.assertListEqual([self.test_filename], sapcar_archive.files_names) - self.assertListEqual([self.test_filename], sapcar_archive.files.keys()) + if six.PY2: + self.assertListEqual([self.test_filename], sapcar_archive.files.keys()) + else: + self.assertListEqual([self.test_filename], list(sapcar_archive.files.keys())) af = sapcar_archive.open(self.test_filename) self.assertEqual(self.test_string, af.read()) @@ -109,7 +113,10 @@ def test_sapcar_archive_add_file(self): self.assertEqual(2, len(ar.files)) self.assertEqual(2, len(ar.files_names)) self.assertListEqual([self.test_filename, self.test_filename+"two"], ar.files_names) - self.assertListEqual([self.test_filename, self.test_filename+"two"], ar.files.keys()) + if six.PY2: + self.assertListEqual([self.test_filename, self.test_filename+"two"], ar.files.keys()) + else: + self.assertListEqual([self.test_filename, self.test_filename+"two"], list(ar.files.keys())) for filename in [self.test_filename, self.test_filename+"two"]: af = ar.open(filename) From b3816bca95d2cf402fd79c63c03af2c6da3d2bc4 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Fri, 6 Jul 2018 15:55:53 +0300 Subject: [PATCH 16/74] Convert comment-like string literals to comments --- pysap/SAPCAR.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/pysap/SAPCAR.py b/pysap/SAPCAR.py index 0ab2222..6ade5e4 100644 --- a/pysap/SAPCAR.py +++ b/pysap/SAPCAR.py @@ -111,17 +111,17 @@ class SAPCARCompressedBlobFormat(PacketNoPadded): ] +# SAP CAR compressed end of data block SAPCAR_BLOCK_TYPE_COMPRESSED_LAST = b"ED" -"""SAP CAR compressed end of data block""" +# SAP CAR compressed block SAPCAR_BLOCK_TYPE_COMPRESSED = b"DA" -"""SAP CAR compressed block""" +# SAP CAR uncompressed end of data block SAPCAR_BLOCK_TYPE_UNCOMPRESSED_LAST = b"UE" -"""SAP CAR uncompressed end of data block""" +# SAP CAR uncompressed block SAPCAR_BLOCK_TYPE_UNCOMPRESSED = b"UD" -"""SAP CAR uncompressed block""" class SAPCARCompressedBlockFormat(PacketNoPadded): @@ -152,27 +152,27 @@ def sapcar_is_last_block(packet): return packet.type in [SAPCAR_BLOCK_TYPE_COMPRESSED_LAST, SAPCAR_BLOCK_TYPE_UNCOMPRESSED_LAST] +# SAP CAR regular file string SAPCAR_TYPE_FILE = b"RG" -"""SAP CAR regular file string""" +# SAP CAR directory string SAPCAR_TYPE_DIR = b"DR" -"""SAP CAR directory string""" +# SAP CAR Windows short cut string SAPCAR_TYPE_SHORTCUT = b"SC" -"""SAP CAR Windows short cut string""" +# SAP CAR Unix soft link string SAPCAR_TYPE_LINK = b"LK" -"""SAP CAR Unix soft link string""" +# SAP CAR AS400 save file string SAPCAR_TYPE_AS400 = b"SV" -"""SAP CAR AS400 save file string""" # Version strings are unicode instead of byte strings in order to avoid constant .decode() and .encode() calls +# SAP CAR file format version 2.00 string SAPCAR_VERSION_200 = "2.00" -"""SAP CAR file format version 2.00 string""" +# SAP CAR file format version 2.01 string SAPCAR_VERSION_201 = "2.01" -"""SAP CAR file format version 2.01 string""" class SAPCARArchiveFilev200Format(PacketNoPadded): @@ -281,18 +281,17 @@ class SAPCARArchiveFilev201Format(SAPCARArchiveFilev200Format): is_filename_null_terminated = True +# SAP CAR archive header magic string standard SAPCAR_HEADER_MAGIC_STRING_STANDARD = b"CAR\x20" -"""SAP CAR archive header magic string standard""" +# SAP CAR archive header magic string backup file SAPCAR_HEADER_MAGIC_STRING_BACKUP = b"CAR\x00" -"""SAP CAR archive header magic string backup file""" - +# SAP CAR file format versions sapcar_archive_file_versions = { SAPCAR_VERSION_200: SAPCARArchiveFilev200Format, SAPCAR_VERSION_201: SAPCARArchiveFilev201Format, } -"""SAP CAR file format versions""" class SAPCARArchiveFormat(Packet): From 80e6a105b755b59627603000a978da9349abb35f Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Fri, 6 Jul 2018 11:17:14 -0300 Subject: [PATCH 17/74] Don't stop build if flake8 find errors --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5a8b163..a2af2bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,7 +66,8 @@ install: - ${PIP} install -r requirements-docs.txt - ${PIP} install -r requirements-optional.txt # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics + # XXX: Disabled stopping the build while working on porting to Python2/3 + - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exit-zero # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics From 5aad8b1450377f313a7ff38b6a6aa5f0e98ade52 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Fri, 6 Jul 2018 11:38:01 -0300 Subject: [PATCH 18/74] Removed resource warning on unittest --- tests/sapcar_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/sapcar_test.py b/tests/sapcar_test.py index f5b2e55..cf8ebd3 100755 --- a/tests/sapcar_test.py +++ b/tests/sapcar_test.py @@ -134,6 +134,8 @@ def test_sapcar_archive_add_file(self): self.assertEqual(self.test_string, af.read()) af.close() + ar.close() + def test_sapcar_archive_file_from_file(self): """Test SAP CAR archive file object construction from file using the original name and a different one""" From b25ec84bb7f863962d5a985daeb6ac2ef2f2e205 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 19 Jul 2018 13:19:53 +0300 Subject: [PATCH 19/74] Rename tests to follow Python naming convention Also changed test_suite in setup.py to "tests", let the unit test framework do the discovery automatically. --- setup.py | 2 +- tests/__init__.py | 2 +- tests/{pysapcompress_test.py => test_pysapcompress.py} | 0 tests/{sapcar_test.py => test_sapcar.py} | 0 tests/{sapcredv2_test.py => test_sapcredv2.py} | 0 tests/{sapdiag_test.py => test_sapdiag.py} | 0 tests/{sapni_test.py => test_sapni.py} | 0 tests/{sappse_test.py => test_sappse.py} | 0 tests/{saprouter_test.py => test_saprouter.py} | 0 9 files changed, 2 insertions(+), 2 deletions(-) mode change 100755 => 100644 tests/__init__.py rename tests/{pysapcompress_test.py => test_pysapcompress.py} (100%) rename tests/{sapcar_test.py => test_sapcar.py} (100%) rename tests/{sapcredv2_test.py => test_sapcredv2.py} (100%) rename tests/{sapdiag_test.py => test_sapdiag.py} (100%) rename tests/{sapni_test.py => test_sapni.py} (100%) rename tests/{sappse_test.py => test_sappse.py} (100%) rename tests/{saprouter_test.py => test_saprouter.py} (100%) diff --git a/setup.py b/setup.py index d7ba080..ffc93e4 100755 --- a/setup.py +++ b/setup.py @@ -116,7 +116,7 @@ def run(self): scripts=['bin/pysapcar', 'bin/pysapgenpse'], # Tests command - test_suite='tests.test_suite', + test_suite='tests', # Documentation commands cmdclass={'doc': APIDocumentationCommand, diff --git a/tests/__init__.py b/tests/__init__.py old mode 100755 new mode 100644 index 6a9ddaa..96f9789 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -22,7 +22,7 @@ import unittest -test_suite = unittest.defaultTestLoader.discover('.', '*_test.py') +test_suite = unittest.defaultTestLoader.discover('.') if __name__ == '__main__': diff --git a/tests/pysapcompress_test.py b/tests/test_pysapcompress.py similarity index 100% rename from tests/pysapcompress_test.py rename to tests/test_pysapcompress.py diff --git a/tests/sapcar_test.py b/tests/test_sapcar.py similarity index 100% rename from tests/sapcar_test.py rename to tests/test_sapcar.py diff --git a/tests/sapcredv2_test.py b/tests/test_sapcredv2.py similarity index 100% rename from tests/sapcredv2_test.py rename to tests/test_sapcredv2.py diff --git a/tests/sapdiag_test.py b/tests/test_sapdiag.py similarity index 100% rename from tests/sapdiag_test.py rename to tests/test_sapdiag.py diff --git a/tests/sapni_test.py b/tests/test_sapni.py similarity index 100% rename from tests/sapni_test.py rename to tests/test_sapni.py diff --git a/tests/sappse_test.py b/tests/test_sappse.py similarity index 100% rename from tests/sappse_test.py rename to tests/test_sappse.py diff --git a/tests/saprouter_test.py b/tests/test_saprouter.py similarity index 100% rename from tests/saprouter_test.py rename to tests/test_saprouter.py From 150d3141f6eacd458f604d8fd55567fa98f39beb Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 19 Jul 2018 13:41:43 +0300 Subject: [PATCH 20/74] Clean up tests/__init__.py Tests should be run using `python -m unittest discover` (or what ever the unit test framework at hand uses) instead of running tests/__init__.py manually. --- tests/__init__.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 96f9789..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,31 +0,0 @@ -# =========== -# pysap - Python library for crafting SAP's network protocols packets -# -# Copyright (C) 2012-2018 by Martin Gallo, Core Security -# -# The library was designed and developed by Martin Gallo from -# Core Security's CoreLabs team. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# ============== - -# Standard imports -import sys -import unittest - - -test_suite = unittest.defaultTestLoader.discover('.') - - -if __name__ == '__main__': - test_runner = unittest.TextTestRunner(verbosity=2, resultclass=unittest.TextTestResult) - result = test_runner.run(test_suite) - sys.exit(not result.wasSuccessful()) From dc889ea1fabc51895cc873ee01aa2fc2f94ec60c Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Fri, 20 Jul 2018 08:36:34 +0300 Subject: [PATCH 21/74] Move pysapcar CLI functionality under pysap module Writing unit tests for CLI tools is a real PITA to begin with, and it becomes even harder when the functionality is under CLI script that does not end with .py as importing it becomes a mess: https://stackoverflow.com/a/2601083 --- bin/pysapcar | 300 +----------------------------------------- pysap/sapcarcli.py | 320 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 322 insertions(+), 298 deletions(-) create mode 100644 pysap/sapcarcli.py diff --git a/bin/pysapcar b/bin/pysapcar index 0237e01..ab14214 100644 --- a/bin/pysapcar +++ b/bin/pysapcar @@ -18,304 +18,8 @@ # GNU General Public License for more details. # ============== -# Standard imports -import logging -from sys import stdin -from os import makedirs, utime, path -from os import sep as dir_separator -from argparse import ArgumentParser -# Custom imports -import pysap -from pysapcompress import DecompressError -from pysap.SAPCAR import SAPCARArchive, SAPCARInvalidChecksumException, SAPCARInvalidFileException - - -# Try to import OS-dependent functions -try: - from os import chmod, fchmod -except ImportError: - chmod = fchmod = None - - -pysapcar_usage = """ - -create archive with specified files: -pysapcar -c[v][f archive] [file1 file2 [/n filename] ...] - -list the contents of an archive: -pysapcar -t[v][f archive] [file1 file2 ...] - -extract files from an archive: -pysapcar -x[v][f archive] [-o outdir] [file1 file2 ...] - -append files to an archive: -pysapcar -a[v][f archive] [file1 file2 [/n filename] ...] - -""" - - -class PySAPCAR(object): - - # Private attributes - _logger = None - - # Instance attributes - mode = None - log_level = None - archive_fd = None - - @staticmethod - def parse_options(): - """Parses command-line options. - """ - - description = "Basic and experimental implementation of SAPCAR archive format." - - epilog = "pysap %(version)s - %(url)s - %(repo)s" % {"version": pysap.__version__, - "url": pysap.__url__, - "repo": pysap.__repo__} - - parser = ArgumentParser(usage=pysapcar_usage, description=description, epilog=epilog) - - # Commands - parser.add_argument("-c", dest="create", action="store_true", help="Create archive with specified files") - parser.add_argument("-x", dest="extract", action="store_true", help="Extract files from an archive") - parser.add_argument("-t", dest="list", action="store_true", help="List the contents of an archive") - parser.add_argument("-a", dest="append", action="store_true", help="Append files to an archive") - parser.add_argument("-f", dest="filename", help="Archive filename", metavar="FILE") - parser.add_argument("-o", dest="outdir", help="Path to directory where to extract files") - - misc = parser.add_argument_group("Misc options") - misc.add_argument("-v", dest="verbose", action="count", help="Verbose output") - misc.add_argument("-e", "--enforce-checksum", dest="enforce_checksum", action="store_true", - help="Whether the checksum validation is enforced. A file with an invalid checksum won't be " - "extracted. When not set, only a warning would be thrown if checksum is invalid.") - misc.add_argument("-b", "--break-on-error", dest="break_on_error", action="store_true", - help="Whether the extraction would continue if an error is identified.") - - (options, args) = parser.parse_known_args() - - return options, args - - @property - def logger(self): - """Sets the logger of the cli tool. - """ - if self._logger is None: - self._logger = logging.getLogger("pysapcar") - self._logger.setLevel(self.log_level + 1) - self._logger.addHandler(logging.StreamHandler()) - return self._logger - - def main(self): - """Main routine for parsing options and dispatch desired action. - """ - options, args = self.parse_options() - - # Set the verbosity - self.log_level = options.verbose or 0 - - self.logger.info("pysapcar version: %s", pysap.__version__) - - # Check the mode the archive file should be opened - if options.list or options.extract: - self.mode = "r" - elif options.create: - self.mode = "w" - elif options.append: - self.mode = "r+" - else: # default to read mode - self.mode = "r" - - # Opens the input/output file - self.archive_fd = None - if options.filename: - try: - self.archive_fd = open(options.filename, self.mode) - except IOError as e: - self.logger.error("pysapcar: error opening '%s' (%s)" % (options.filename, e.strerror)) - return - else: - self.archive_fd = stdin - - # Execute the action - try: - if options.create or options.append: - self.append(options, args) - elif options.list: - self.list(options, args) - elif options.extract: - self.extract(options, args) - finally: - self.archive_fd.close() - - def open_archive(self): - """Opens the archive file to work with it and returns - the SAP Car Archive object. - """ - try: - sapcar = SAPCARArchive(self.archive_fd, mode=self.mode) - self.logger.info("pysapcar: Processing archive '%s' (version %s)", self.archive_fd.name, sapcar.version) - except Exception as e: - self.logger.error("pysapcar: Error processing archive '%s' (%s)", self.archive_fd.name, e.message) - return None - return sapcar - - @staticmethod - def target_files(filenames, target_filenames=None): - """Generates the list of files to work on. It calculates - the intersection between the file names selected in - command-line and the ones in the archive to work on. - - :param filenames: filenames in the archive - :param target_filenames: filenames to work on - - :return: filename - """ - files = set(filenames) - if target_filenames: - files = files.intersection(set(target_filenames)) - - for filename in files: - yield filename - - def append(self, options, args): - """Appends a file to the archive file. - """ - - if len(args) < 1: - self.logger.error("pysapcar: no files specified for appending") - return - - # Open the archive file - sapcar = self.open_archive() - if not sapcar: - return - - while len(args): - filename = filename_in_archive = args.pop(0) - - self.logger.debug(args) - if len(args) >= 2 and args[0] == "/n": - args.pop(0) - filename_in_archive = args.pop(0) - - sapcar.add_file(filename, archive_filename=filename_in_archive) - if filename != filename_in_archive: - self.logger.info("d %s (original name %s)", filename_in_archive, filename) - else: - self.logger.info("d %s", filename) - - sapcar.write() - - def list(self, options, args): - """List files inside the archive file and print their - attributes: permissions, size, timestamp and filename. - """ - # Open the archive file - sapcar = self.open_archive() - if not sapcar: - return - # Print the info of each file - for filename in self.target_files(sapcar.files_names, args): - fil = sapcar.files[filename] - self.logger.info("{} {:>10} {} {}".format(fil.permissions, fil.size, fil.timestamp, fil.filename)) - - def extract(self, options, args): - """Extract files from the archive file. - """ - CONTINUE = 1 - SKIP = 2 - STOP = 3 - - # Open the archive file - sapcar = self.open_archive() - if not sapcar: - return - - # Warn if permissions can't be set - if not chmod: - self.logger.warning("pysapcar: Setting extracted files permissions not implemented in this platform") - - # Extract each file in the archive - no = 0 - for filename in self.target_files(sapcar.files_names, args): - flag = CONTINUE - fil = sapcar.files[filename] - filename = path.normpath(filename.replace("\x00", "")) # Take out null bytes if found - if options.outdir: - # Have to strip directory separator from the beginning of the file name, because path.join disregards - # all previous components if any of the following components is an absolute path - filename = path.join(path.normpath(options.outdir), filename.lstrip(dir_separator)) - - if fil.is_directory(): - # If the directory doesn't exist, create it and set permissions and timestamp - if not path.exists(filename): - makedirs(filename) - if chmod: - chmod(filename, fil.perm_mode) - utime(filename, (fil.timestamp_raw, fil.timestamp_raw)) - self.logger.info("d %s", filename) - no += 1 - - elif fil.is_file(): - # If the file references a directory that is not there, create it first - file_dirname = path.dirname(filename) - if file_dirname and not path.exists(file_dirname): - # mkdir barfs if archive contains /foo/bar/bash but not /foo/bar directory. - # makedirs creates intermediate directories as well - makedirs(file_dirname) - self.logger.info("d %s", file_dirname) - - # Try to extract the file and handle potential errors - try: - data = fil.open(enforce_checksum=options.enforce_checksum).read() - except (SAPCARInvalidFileException, DecompressError) as e: - self.logger.error("pysapcar: Invalid SAP CAR file '%s' (%s)", self.archive_fd.name, e.message) - if options.break_on_error: - flag = STOP - else: - flag = SKIP - except SAPCARInvalidChecksumException: - self.logger.error("pysapcar: Invalid checksum found for file '%s'", fil.filename) - if options.enforce_checksum: - flag = STOP - - # Check the result before starting to write the file - if flag == SKIP: - self.logger.info("pysapcar: Skipping execution of file '%s'", fil.filename) - continue - elif flag == STOP: - self.logger.info("pysapcar: Stopping extraction") - break - - # Write the new file and set permissions - try: - with open(filename, "wb") as new_file: - new_file.write(data) - if fchmod: - fchmod(new_file.fileno(), fil.perm_mode) - except IOError as e: - self.logger.error("pysapcar: Failed to extract file '%s', reason: %s", filename, e.strerror) - if options.break_on_error: - break - - # Set the timestamp - utime(filename, (fil.timestamp_raw, fil.timestamp_raw)) - - # If this path is reached and checksum is not valid, means checksum is not enforced, so we should warn - # only. - if not fil.check_checksum(): - self.logger.warning("pysapcar: checksum error in '%s' !", filename) - - self.logger.info("d %s", filename) - no += 1 - else: - self.logger.warning("pysapcar: Invalid file type '%s'", filename) - - self.logger.info("pysapcar: %d file(s) processed", no) +from pysap.sapcarcli import PySAPCAR if __name__ == "__main__": - pysapcar = PySAPCAR() - pysapcar.main() + PySAPCAR().main() diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py new file mode 100644 index 0000000..9759c54 --- /dev/null +++ b/pysap/sapcarcli.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python +# =========== +# pysap - Python library for crafting SAP's network protocols packets +# +# Copyright (C) 2012-2018 by Martin Gallo, Core Security +# +# The library was designed and developed by Martin Gallo from +# Core Security's CoreLabs team. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# ============== + +# Standard imports +import logging +from sys import stdin +from os import makedirs, utime, path +from os import sep as dir_separator +from argparse import ArgumentParser +# Custom imports +import pysap +from pysapcompress import DecompressError +from pysap.SAPCAR import SAPCARArchive, SAPCARInvalidChecksumException, SAPCARInvalidFileException + + +# Try to import OS-dependent functions +try: + from os import chmod, fchmod +except ImportError: + chmod = fchmod = None + + +pysapcar_usage = """ + +create archive with specified files: +pysapcar -c[v][f archive] [file1 file2 [/n filename] ...] + +list the contents of an archive: +pysapcar -t[v][f archive] [file1 file2 ...] + +extract files from an archive: +pysapcar -x[v][f archive] [-o outdir] [file1 file2 ...] + +append files to an archive: +pysapcar -a[v][f archive] [file1 file2 [/n filename] ...] + +""" + + +class PySAPCAR(object): + + # Private attributes + _logger = None + + # Instance attributes + mode = None + log_level = None + archive_fd = None + + @staticmethod + def parse_options(): + """Parses command-line options. + """ + + description = "Basic and experimental implementation of SAPCAR archive format." + + epilog = "pysap %(version)s - %(url)s - %(repo)s" % {"version": pysap.__version__, + "url": pysap.__url__, + "repo": pysap.__repo__} + + parser = ArgumentParser(usage=pysapcar_usage, description=description, epilog=epilog) + + # Commands + parser.add_argument("-c", dest="create", action="store_true", help="Create archive with specified files") + parser.add_argument("-x", dest="extract", action="store_true", help="Extract files from an archive") + parser.add_argument("-t", dest="list", action="store_true", help="List the contents of an archive") + parser.add_argument("-a", dest="append", action="store_true", help="Append files to an archive") + parser.add_argument("-f", dest="filename", help="Archive filename", metavar="FILE") + parser.add_argument("-o", dest="outdir", help="Path to directory where to extract files") + + misc = parser.add_argument_group("Misc options") + misc.add_argument("-v", dest="verbose", action="count", help="Verbose output") + misc.add_argument("-e", "--enforce-checksum", dest="enforce_checksum", action="store_true", + help="Whether the checksum validation is enforced. A file with an invalid checksum won't be " + "extracted. When not set, only a warning would be thrown if checksum is invalid.") + misc.add_argument("-b", "--break-on-error", dest="break_on_error", action="store_true", + help="Whether the extraction would continue if an error is identified.") + + (options, args) = parser.parse_known_args() + + return options, args + + @property + def logger(self): + """Sets the logger of the cli tool. + """ + if self._logger is None: + self._logger = logging.getLogger("pysapcar") + self._logger.setLevel(self.log_level + 1) + self._logger.addHandler(logging.StreamHandler()) + return self._logger + + def main(self): + """Main routine for parsing options and dispatch desired action. + """ + options, args = self.parse_options() + + # Set the verbosity + self.log_level = options.verbose or 0 + + self.logger.info("pysapcar version: %s", pysap.__version__) + + # Check the mode the archive file should be opened + if options.list or options.extract: + self.mode = "r" + elif options.create: + self.mode = "w" + elif options.append: + self.mode = "r+" + else: # default to read mode + self.mode = "r" + + # Opens the input/output file + self.archive_fd = None + if options.filename: + try: + self.archive_fd = open(options.filename, self.mode) + except IOError as e: + self.logger.error("pysapcar: error opening '%s' (%s)" % (options.filename, e.strerror)) + return + else: + self.archive_fd = stdin + + # Execute the action + try: + if options.create or options.append: + self.append(options, args) + elif options.list: + self.list(options, args) + elif options.extract: + self.extract(options, args) + finally: + self.archive_fd.close() + + def open_archive(self): + """Opens the archive file to work with it and returns + the SAP Car Archive object. + """ + try: + sapcar = SAPCARArchive(self.archive_fd, mode=self.mode) + self.logger.info("pysapcar: Processing archive '%s' (version %s)", self.archive_fd.name, sapcar.version) + except Exception as e: + self.logger.error("pysapcar: Error processing archive '%s' (%s)", self.archive_fd.name, e.message) + return None + return sapcar + + @staticmethod + def target_files(filenames, target_filenames=None): + """Generates the list of files to work on. It calculates + the intersection between the file names selected in + command-line and the ones in the archive to work on. + + :param filenames: filenames in the archive + :param target_filenames: filenames to work on + + :return: filename + """ + files = set(filenames) + if target_filenames: + files = files.intersection(set(target_filenames)) + + for filename in files: + yield filename + + def append(self, options, args): + """Appends a file to the archive file. + """ + + if len(args) < 1: + self.logger.error("pysapcar: no files specified for appending") + return + + # Open the archive file + sapcar = self.open_archive() + if not sapcar: + return + + while len(args): + filename = filename_in_archive = args.pop(0) + + self.logger.debug(args) + if len(args) >= 2 and args[0] == "/n": + args.pop(0) + filename_in_archive = args.pop(0) + + sapcar.add_file(filename, archive_filename=filename_in_archive) + if filename != filename_in_archive: + self.logger.info("d %s (original name %s)", filename_in_archive, filename) + else: + self.logger.info("d %s", filename) + + sapcar.write() + + def list(self, options, args): + """List files inside the archive file and print their + attributes: permissions, size, timestamp and filename. + """ + # Open the archive file + sapcar = self.open_archive() + if not sapcar: + return + # Print the info of each file + for filename in self.target_files(sapcar.files_names, args): + fil = sapcar.files[filename] + self.logger.info("{} {:>10} {} {}".format(fil.permissions, fil.size, fil.timestamp, fil.filename)) + + def extract(self, options, args): + """Extract files from the archive file. + """ + CONTINUE = 1 + SKIP = 2 + STOP = 3 + + # Open the archive file + sapcar = self.open_archive() + if not sapcar: + return + + # Warn if permissions can't be set + if not chmod: + self.logger.warning("pysapcar: Setting extracted files permissions not implemented in this platform") + + # Extract each file in the archive + no = 0 + for filename in self.target_files(sapcar.files_names, args): + flag = CONTINUE + fil = sapcar.files[filename] + filename = path.normpath(filename.replace("\x00", "")) # Take out null bytes if found + if options.outdir: + # Have to strip directory separator from the beginning of the file name, because path.join disregards + # all previous components if any of the following components is an absolute path + filename = path.join(path.normpath(options.outdir), filename.lstrip(dir_separator)) + + if fil.is_directory(): + # If the directory doesn't exist, create it and set permissions and timestamp + if not path.exists(filename): + makedirs(filename) + if chmod: + chmod(filename, fil.perm_mode) + utime(filename, (fil.timestamp_raw, fil.timestamp_raw)) + self.logger.info("d %s", filename) + no += 1 + + elif fil.is_file(): + # If the file references a directory that is not there, create it first + file_dirname = path.dirname(filename) + if file_dirname and not path.exists(file_dirname): + # mkdir barfs if archive contains /foo/bar/bash but not /foo/bar directory. + # makedirs creates intermediate directories as well + makedirs(file_dirname) + self.logger.info("d %s", file_dirname) + + # Try to extract the file and handle potential errors + try: + data = fil.open(enforce_checksum=options.enforce_checksum).read() + except (SAPCARInvalidFileException, DecompressError) as e: + self.logger.error("pysapcar: Invalid SAP CAR file '%s' (%s)", self.archive_fd.name, e.message) + if options.break_on_error: + flag = STOP + else: + flag = SKIP + except SAPCARInvalidChecksumException: + self.logger.error("pysapcar: Invalid checksum found for file '%s'", fil.filename) + if options.enforce_checksum: + flag = STOP + + # Check the result before starting to write the file + if flag == SKIP: + self.logger.info("pysapcar: Skipping execution of file '%s'", fil.filename) + continue + elif flag == STOP: + self.logger.info("pysapcar: Stopping extraction") + break + + # Write the new file and set permissions + try: + with open(filename, "wb") as new_file: + new_file.write(data) + if fchmod: + fchmod(new_file.fileno(), fil.perm_mode) + except IOError as e: + self.logger.error("pysapcar: Failed to extract file '%s', reason: %s", filename, e.strerror) + if options.break_on_error: + break + + # Set the timestamp + utime(filename, (fil.timestamp_raw, fil.timestamp_raw)) + + # If this path is reached and checksum is not valid, means checksum is not enforced, so we should warn + # only. + if not fil.check_checksum(): + self.logger.warning("pysapcar: checksum error in '%s' !", filename) + + self.logger.info("d %s", filename) + no += 1 + else: + self.logger.warning("pysapcar: Invalid file type '%s'", filename) + + self.logger.info("pysapcar: %d file(s) processed", no) + + +if __name__ == "__main__": + PySAPCAR().main() From 30bcf0d18f7c634648bb13441e5cdccc51d71d26 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Fri, 20 Jul 2018 08:46:30 +0300 Subject: [PATCH 22/74] Add __init__ method for PySAPCAR Not strictly necessary, but more Pythonic --- pysap/sapcarcli.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index 9759c54..0d962b2 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -56,13 +56,14 @@ class PySAPCAR(object): - # Private attributes - _logger = None + def __init__(self): + # Private attributes + self._logger = None - # Instance attributes - mode = None - log_level = None - archive_fd = None + # Instance attributes + self.mode = None + self.log_level = None + self.archive_fd = None @staticmethod def parse_options(): From 3a94974ebe6c1d5b8cb0c6f236e9fd1434c4d870 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Tue, 24 Jul 2018 09:44:15 +0300 Subject: [PATCH 23/74] Add requirements file for tests Mock is part of unittest lib in Py3, but needs to be fetched from PIP on Py2. --- requirements-tests.txt | 1 + setup.py | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 requirements-tests.txt diff --git a/requirements-tests.txt b/requirements-tests.txt new file mode 100644 index 0000000..455de9f --- /dev/null +++ b/requirements-tests.txt @@ -0,0 +1 @@ +mock==2.0.0; python_version < '3' diff --git a/setup.py b/setup.py index ffc93e4..ff74a41 100755 --- a/setup.py +++ b/setup.py @@ -125,6 +125,8 @@ def run(self): # Requirements install_requires=open('requirements.txt').read().splitlines(), + tests_require=open('requirements-tests.txt').read().splitlines(), + # Optional requirements for docs and some examples extras_require={"docs": open('requirements-docs.txt').read().splitlines(), "examples": open('requirements-optional.txt').read().splitlines()}, From 1c8284522a179f1df5b4842e24123016990684c0 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Tue, 24 Jul 2018 09:49:18 +0300 Subject: [PATCH 24/74] Add tests for sapcarcli open_archive --- tests/test_sapcarcli.py | 72 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tests/test_sapcarcli.py diff --git a/tests/test_sapcarcli.py b/tests/test_sapcarcli.py new file mode 100644 index 0000000..f0f9021 --- /dev/null +++ b/tests/test_sapcarcli.py @@ -0,0 +1,72 @@ +# =========== +# pysap - Python library for crafting SAP's network protocols packets +# +# Copyright (C) 2012-2018 by Martin Gallo, Core Security +# +# The library was designed and developed by Martin Gallo from +# Core Security's CoreLabs team. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# ============== + +# Standard imports +from __future__ import unicode_literals, absolute_import +import unittest + +try: + from unittest import mock +except ImportError: + import mock + +from tests.utils import data_filename +from pysap.sapcarcli import PySAPCAR +from pysap.SAPCAR import SAPCARArchive + +class PySAPCARCLITest(unittest.TestCase): + def setUp(self): + self.cli = PySAPCAR() + self.cli.mode = "r" + self.cli.archive_fd = open(data_filename("car201_test_string.sar"), "rb") + self.cli.logger.disabled = True # Mute logging when executing unit tests + + def tearDown(self): + try: + self.cli.archive_fd.close() + except Exception: + pass + + def test_open_archive_fd_not_set(self): + temp = self.cli.archive_fd # Backup test archive file handle + self.cli.archive_fd = None + self.assertIsNone(self.cli.open_archive()) + self.cli.archive_fd = temp + + def test_open_archive_raises_exception(self): + # While SAPCARArchive is defined in pysap.SAPCAR, it's looked up in sapcarcli, so we need to patch it there + # See https://docs.python.org/3/library/unittest.mock.html#where-to-patch for details + with mock.patch("pysap.sapcarcli.SAPCARArchive") as mock_archive: + mock_archive.side_effect = Exception("unit test exception") + self.assertIsNone(self.cli.open_archive()) + mock_archive.assert_called_once_with(self.cli.archive_fd, mode=self.cli.mode) + + def test_open_archive_succeeds(self): + self.assertIsInstance(self.cli.open_archive(), SAPCARArchive) + + +def test_suite(): + loader = unittest.TestLoader() + suite = unittest.TestSuite() + suite.addTest(loader.loadTestsFromTestCase(PySAPCARCLITest)) + return suite + + +if __name__ == "__main__": + unittest.TextTestRunner(verbosity=2).run(test_suite()) From a60143d298b77ebe25d07493e4b32d63c9c77b52 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Tue, 24 Jul 2018 09:50:05 +0300 Subject: [PATCH 25/74] Set log level to 0 by default Otherwise tests crash as logger tries to add 1 to None. --- pysap/sapcarcli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index 0d962b2..a945b72 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -62,7 +62,7 @@ def __init__(self): # Instance attributes self.mode = None - self.log_level = None + self.log_level = 0 self.archive_fd = None @staticmethod From 8223e7ed4a21c6015a67a3b43b64ebefd8cdc941 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Tue, 24 Jul 2018 09:53:04 +0300 Subject: [PATCH 26/74] Check that archive_fd is set before opening Otherwise error handling raises AttributeError as None has no name. Also removed some redundant lines, like resetting archive_fd in main loop and returning None. --- pysap/sapcarcli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index a945b72..274cd69 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -129,7 +129,6 @@ def main(self): self.mode = "r" # Opens the input/output file - self.archive_fd = None if options.filename: try: self.archive_fd = open(options.filename, self.mode) @@ -154,12 +153,15 @@ def open_archive(self): """Opens the archive file to work with it and returns the SAP Car Archive object. """ + if not self.archive_fd: + self.logger.critical("pysapcar: Trying to open archive file before setting it, should not happen") + return try: sapcar = SAPCARArchive(self.archive_fd, mode=self.mode) self.logger.info("pysapcar: Processing archive '%s' (version %s)", self.archive_fd.name, sapcar.version) except Exception as e: self.logger.error("pysapcar: Error processing archive '%s' (%s)", self.archive_fd.name, e.message) - return None + return return sapcar @staticmethod From ceef061cdd949f399ff9eabb779317ec14bf7bdd Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Tue, 24 Jul 2018 09:57:28 +0300 Subject: [PATCH 27/74] Log plain error instead of error.message Error.message has been deprecated since Py2.6 and removed in Py3. Logging just the error object should work just fine. --- pysap/sapcarcli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index 274cd69..8cc82c6 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -160,7 +160,7 @@ def open_archive(self): sapcar = SAPCARArchive(self.archive_fd, mode=self.mode) self.logger.info("pysapcar: Processing archive '%s' (version %s)", self.archive_fd.name, sapcar.version) except Exception as e: - self.logger.error("pysapcar: Error processing archive '%s' (%s)", self.archive_fd.name, e.message) + self.logger.error("pysapcar: Error processing archive '%s' (%s)", self.archive_fd.name, e) return return sapcar @@ -274,7 +274,7 @@ def extract(self, options, args): try: data = fil.open(enforce_checksum=options.enforce_checksum).read() except (SAPCARInvalidFileException, DecompressError) as e: - self.logger.error("pysapcar: Invalid SAP CAR file '%s' (%s)", self.archive_fd.name, e.message) + self.logger.error("pysapcar: Invalid SAP CAR file '%s' (%s)", self.archive_fd.name, e) if options.break_on_error: flag = STOP else: From b617924a6f491f5be35e9dd68a4670a3f024ea52 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Tue, 24 Jul 2018 12:29:14 +0300 Subject: [PATCH 28/74] Ensure iteration order in target_files method Testing this and list is tricky if list generation order is random. According to %timeit, sorting a set of 100k entries only adds about 10ns to execution time. Of course it adds a bit of memory consumption, but I think it's negligible in real world use cases. --- pysap/sapcarcli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index 8cc82c6..d3d8623 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -179,7 +179,7 @@ def target_files(filenames, target_filenames=None): if target_filenames: files = files.intersection(set(target_filenames)) - for filename in files: + for filename in sorted(files): # Ensure iteration order yield filename def append(self, options, args): From a9446936c4cd7c13208983c4a2b0ae0c01fc2a93 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Tue, 24 Jul 2018 12:34:36 +0300 Subject: [PATCH 29/74] Add tests for target_files method --- tests/test_sapcarcli.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_sapcarcli.py b/tests/test_sapcarcli.py index f0f9021..08275e0 100644 --- a/tests/test_sapcarcli.py +++ b/tests/test_sapcarcli.py @@ -60,6 +60,14 @@ def test_open_archive_raises_exception(self): def test_open_archive_succeeds(self): self.assertIsInstance(self.cli.open_archive(), SAPCARArchive) + def test_target_files_no_kwargs(self): + names = sorted(["list", "of", "test", "names"]) + self.assertEqual(names, [n for n in self.cli.target_files(names)]) + + def test_target_files_with_kwargs(self): + names = sorted(["list", "of", "test", "names"]) + targets = sorted(["list", "names", "blah"]) + self.assertEqual(["list", "names"], [n for n in self.cli.target_files(names, targets)]) def test_suite(): loader = unittest.TestLoader() From 633681006a2373b57aed21e16c0d58b483afa5cf Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Tue, 24 Jul 2018 14:38:53 +0300 Subject: [PATCH 30/74] Use logger internal string formatting in list Doesn't really matter in this use case but just for the sake of consistency and being more Pythonic. --- pysap/sapcarcli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index d3d8623..11ec12a 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -222,7 +222,7 @@ def list(self, options, args): # Print the info of each file for filename in self.target_files(sapcar.files_names, args): fil = sapcar.files[filename] - self.logger.info("{} {:>10} {} {}".format(fil.permissions, fil.size, fil.timestamp, fil.filename)) + self.logger.info("%s %10d %s %s", fil.permissions, fil.size, fil.timestamp, fil.filename) def extract(self, options, args): """Extract files from the archive file. From af6c2083a07702c842d92a273672b34ebec8d448 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Tue, 24 Jul 2018 12:02:34 +0300 Subject: [PATCH 31/74] Add tests for PySAPCAR.list method Also added testfixtures package to test requirements, as it offers handy stuff like logging output checking, which is the correct way of making sure the output is correct instead of just checking calls made for logger. --- requirements-tests.txt | 1 + tests/test_sapcarcli.py | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 455de9f..c8e2159 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1 +1,2 @@ +testfixtures==6.2.0 mock==2.0.0; python_version < '3' diff --git a/tests/test_sapcarcli.py b/tests/test_sapcarcli.py index 08275e0..6a7517f 100644 --- a/tests/test_sapcarcli.py +++ b/tests/test_sapcarcli.py @@ -26,6 +26,7 @@ except ImportError: import mock +from testfixtures import LogCapture from tests.utils import data_filename from pysap.sapcarcli import PySAPCAR from pysap.SAPCAR import SAPCARArchive @@ -35,13 +36,15 @@ def setUp(self): self.cli = PySAPCAR() self.cli.mode = "r" self.cli.archive_fd = open(data_filename("car201_test_string.sar"), "rb") - self.cli.logger.disabled = True # Mute logging when executing unit tests + self.cli.logger.handlers = [] # Mute logging while running unit tests + self.log = LogCapture() # Enable logger output checking def tearDown(self): try: self.cli.archive_fd.close() except Exception: pass + self.log.uninstall() def test_open_archive_fd_not_set(self): temp = self.cli.archive_fd # Backup test archive file handle @@ -69,6 +72,36 @@ def test_target_files_with_kwargs(self): targets = sorted(["list", "names", "blah"]) self.assertEqual(["list", "names"], [n for n in self.cli.target_files(names, targets)]) + def test_list_open_fails(self): + with mock.patch.object(self.cli, "open_archive", return_value=None): + self.assertIsNone(self.cli.list(None, None)) + + def test_list_succeeds(self): + archive_attrs = { + "files_names": ["test_string.txt", "blah.so"], + "files": { + "test_string.txt": mock.MagicMock(**{ + "permissions": "-rw-rw-r--", + "size": 43, + "timestamp": "01 Dec 2015 22:48", + "filename": "test_string.txt" + }), + "blah.so": mock.MagicMock(**{ + "permissions": "-rwxrwxr-x", + "size": 1243, + "timestamp": "01 Dec 2017 13:37", + "filename": "blah.so" + }) + } + } + mock_archive = mock.MagicMock(**archive_attrs) + with mock.patch.object(self.cli, "open_archive", return_value=mock_archive): + self.cli.list(None, None) + messages = ("{} {:>10} {} {}".format(fil.permissions, fil.size, fil.timestamp, fil.filename) + for fil in archive_attrs["files"].values()) + logs = (("pysapcar", "INFO", message) for message in messages) + self.log.check_present(*logs, order_matters=False) + def test_suite(): loader = unittest.TestLoader() suite = unittest.TestSuite() From af70e3cb79d3580ac8efa9d9a862ae55491ed5e1 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Wed, 25 Jul 2018 15:03:12 +0300 Subject: [PATCH 32/74] Add tests for PySAPCAR.append --- tests/test_sapcarcli.py | 47 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_sapcarcli.py b/tests/test_sapcarcli.py index 6a7517f..add1a2d 100644 --- a/tests/test_sapcarcli.py +++ b/tests/test_sapcarcli.py @@ -102,6 +102,53 @@ def test_list_succeeds(self): logs = (("pysapcar", "INFO", message) for message in messages) self.log.check_present(*logs, order_matters=False) + def test_append_no_args(self): + self.cli.append(None, []) + self.log.check_present(("pysapcar", "ERROR", "pysapcar: no files specified for appending")) + + def test_append_open_fails(self): + with mock.patch.object(self.cli, "open_archive", return_value=None): + self.assertIsNone(self.cli.append(None, ["blah"])) + self.log.check() # Check that there's no log messages + + def test_append_no_renaming(self): + names = ["list", "of", "test", "names"] + mock_sar = mock.Mock(spec=SAPCARArchive) + # Because of pass by reference and lazy string formatting in logging, we can't actually check that args are + # being printed correctly. Or we could if we logged a copy of args or str(args) in production code, but it would + # beat the point of pass by reference and lazy string formatting, so let's not. Left the generator here because + # it's pretty in it's obscurity. And because someone (probably future me) will probably bang one's head + # against the same wall I just did before coming to the same realization I just did, so here's a little + # something for the next poor soul. May it make your head-against-the-wall session shorter than mine was. + # debugs = (("pysapcar", "DEBUG", str(names[i + 1:])) for i in range(len(names))) + infos = (("pysapcar", "INFO", "d {}".format(name)) for name in names) + calls = [mock.call.add_file(name, archive_filename=name) for name in names] + calls.append(mock.call.write()) + with mock.patch.object(self.cli, "open_archive", return_value=mock_sar): + # Pass copy of names so generator works correctly + self.cli.append(None, names[:]) + # self.log.check_present(*debugs) + self.log.check_present(*infos) + # For some reason mock_sar.assert_has_calls(calls) fails, even though this passes... + self.assertEqual(calls, mock_sar.mock_calls) + + def test_append_with_renaming(self): + names = ["test", "/n", "blah", "test", "test", "/n", "blah2"] + archive_names = [("test", "blah"), ("test", "blah2")] + # List instead of generator, as we need to insert one line + infos = [("pysapcar", "INFO", "d {} (original name {})".format(archive_name, name)) + for name, archive_name in archive_names] + infos.insert(1, ("pysapcar", "INFO", "d {}".format("test"))) + mock_sar = mock.Mock(spec=SAPCARArchive) + calls = [mock.call.add_file(name, archive_filename=archive_name) for name, archive_name in archive_names] + calls.insert(1, mock.call.add_file("test", archive_filename="test")) + calls.append(mock.call.write()) + with mock.patch.object(self.cli, "open_archive", return_value=mock_sar): + self.cli.append(None, names[:]) + self.log.check_present(*infos) + # For some reason mock_sar.assert_has_calls(calls) fails, even though this passes... + self.assertEqual(calls, mock_sar.mock_calls) + def test_suite(): loader = unittest.TestLoader() suite = unittest.TestSuite() From c1278e25c604ed91d99529df7c4fc954a20a3be1 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 2 Aug 2018 12:34:13 +0300 Subject: [PATCH 33/74] Reword log message --- pysap/sapcarcli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index 11ec12a..5cf9bc5 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -286,7 +286,7 @@ def extract(self, options, args): # Check the result before starting to write the file if flag == SKIP: - self.logger.info("pysapcar: Skipping execution of file '%s'", fil.filename) + self.logger.info("pysapcar: Skipping extraction of file '%s'", fil.filename) continue elif flag == STOP: self.logger.info("pysapcar: Stopping extraction") From f5c53fff117a3b90f3dd083586aa9ac48e62be74 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 2 Aug 2018 12:43:20 +0300 Subject: [PATCH 34/74] Add error handling for makedirs commands Also use error number to determine whether the directory exists rather than path.exists. --- pysap/sapcarcli.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index 5cf9bc5..6c57a8f 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -252,9 +252,19 @@ def extract(self, options, args): filename = path.join(path.normpath(options.outdir), filename.lstrip(dir_separator)) if fil.is_directory(): - # If the directory doesn't exist, create it and set permissions and timestamp - if not path.exists(filename): + try: makedirs(filename) + except OSError as e: + # errno 17 == File exists + if e.errno != 17: + self.logger.error("pysapcar: Could not create directory '%s' (%s)", filename, e) + if options.break_on_error: + self.logger.info("pysapcar: Stopping extraction") + break + self.logger.info("pysapcar: Skipping extraction of directory '%s'", filename) + continue + else: + # No need to check for errors on chmod and utime: makedirs would've already raised if chmod: chmod(filename, fil.perm_mode) utime(filename, (fil.timestamp_raw, fil.timestamp_raw)) @@ -264,11 +274,21 @@ def extract(self, options, args): elif fil.is_file(): # If the file references a directory that is not there, create it first file_dirname = path.dirname(filename) - if file_dirname and not path.exists(file_dirname): + if file_dirname: # mkdir barfs if archive contains /foo/bar/bash but not /foo/bar directory. # makedirs creates intermediate directories as well - makedirs(file_dirname) - self.logger.info("d %s", file_dirname) + try: + makedirs(file_dirname) + self.logger.info("d %s", file_dirname) + except OSError as e: + # errno 17 == File exists + if e.errno != 17: + self.logger.error("pysapcar: Could not create intermediate directory '%s' for '%s' (%s)", + file_dirname, filename, e.strerror) + if options.break_on_error: + flag = STOP + else: + flag = SKIP # Try to extract the file and handle potential errors try: From 4fd469f99dfe3f1b87609daaa834d9e05c223646 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 2 Aug 2018 15:52:56 +0300 Subject: [PATCH 35/74] Catch OSError with IOError File opening could go wrong for other reasons as well. Besides, starting with Py3.3 IOError is merged into OSError anyway. --- pysap/sapcarcli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index 6c57a8f..01bfb50 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -132,7 +132,7 @@ def main(self): if options.filename: try: self.archive_fd = open(options.filename, self.mode) - except IOError as e: + except (OSError, IOError) as e: self.logger.error("pysapcar: error opening '%s' (%s)" % (options.filename, e.strerror)) return else: @@ -254,7 +254,7 @@ def extract(self, options, args): if fil.is_directory(): try: makedirs(filename) - except OSError as e: + except (IOError, OSError) as e: # errno 17 == File exists if e.errno != 17: self.logger.error("pysapcar: Could not create directory '%s' (%s)", filename, e) @@ -280,7 +280,7 @@ def extract(self, options, args): try: makedirs(file_dirname) self.logger.info("d %s", file_dirname) - except OSError as e: + except (IOError, OSError) as e: # errno 17 == File exists if e.errno != 17: self.logger.error("pysapcar: Could not create intermediate directory '%s' for '%s' (%s)", @@ -318,7 +318,7 @@ def extract(self, options, args): new_file.write(data) if fchmod: fchmod(new_file.fileno(), fil.perm_mode) - except IOError as e: + except (IOError, OSError) as e: self.logger.error("pysapcar: Failed to extract file '%s', reason: %s", filename, e.strerror) if options.break_on_error: break From 31d24254a5f6fee0e4ce074c87f33c18a8521f8a Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 2 Aug 2018 17:05:05 +0300 Subject: [PATCH 36/74] Tweak checksum enforcement warn / error logic Error should only be shown if checksum is enforced. Also makes sense to have the warning message near the enforcement check rather than at the bottom of extraction loop. --- pysap/sapcarcli.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index 01bfb50..41c5b7c 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -300,9 +300,11 @@ def extract(self, options, args): else: flag = SKIP except SAPCARInvalidChecksumException: - self.logger.error("pysapcar: Invalid checksum found for file '%s'", fil.filename) if options.enforce_checksum: + self.logger.error("pysapcar: Invalid checksum found for file '%s'", fil.filename) flag = STOP + else: + self.logger.warning("pysapcar: checksum error in '%s' !", filename) # Check the result before starting to write the file if flag == SKIP: @@ -326,11 +328,6 @@ def extract(self, options, args): # Set the timestamp utime(filename, (fil.timestamp_raw, fil.timestamp_raw)) - # If this path is reached and checksum is not valid, means checksum is not enforced, so we should warn - # only. - if not fil.check_checksum(): - self.logger.warning("pysapcar: checksum error in '%s' !", filename) - self.logger.info("d %s", filename) no += 1 else: From 8103b0d5f4025ae9064a3f9660ffe0446d207398 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 2 Aug 2018 17:07:48 +0300 Subject: [PATCH 37/74] Fix error handling for writing extracted file utime will raise if permission is denied or file doesn't exist (writing extracted file to disk fails). Writing file to disk -error handler now respects break_on_error flag like extraction. --- pysap/sapcarcli.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index 41c5b7c..7c7f3f0 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -320,13 +320,15 @@ def extract(self, options, args): new_file.write(data) if fchmod: fchmod(new_file.fileno(), fil.perm_mode) + # Set the timestamp + utime(filename, (fil.timestamp_raw, fil.timestamp_raw)) except (IOError, OSError) as e: - self.logger.error("pysapcar: Failed to extract file '%s', reason: %s", filename, e.strerror) + self.logger.error("pysapcar: Failed to extract file '%s', (%s)", filename, e.strerror) if options.break_on_error: + self.logger.info("pysapcar: Stopping extraction") break - - # Set the timestamp - utime(filename, (fil.timestamp_raw, fil.timestamp_raw)) + self.logger.info("pysapcar: Skipping extraction of file '%s'", fil.filename) + continue self.logger.info("d %s", filename) no += 1 From 3efa5976dd666bd7c3e68418750c4a1326fc24e0 Mon Sep 17 00:00:00 2001 From: Oula Kuuva Date: Thu, 2 Aug 2018 18:05:22 +0300 Subject: [PATCH 38/74] Add tests for extract method --- tests/test_sapcarcli.py | 216 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 215 insertions(+), 1 deletion(-) diff --git a/tests/test_sapcarcli.py b/tests/test_sapcarcli.py index add1a2d..d697b16 100644 --- a/tests/test_sapcarcli.py +++ b/tests/test_sapcarcli.py @@ -26,13 +26,39 @@ except ImportError: import mock +from os import path + from testfixtures import LogCapture from tests.utils import data_filename from pysap.sapcarcli import PySAPCAR -from pysap.SAPCAR import SAPCARArchive +from pysap.SAPCAR import SAPCARArchive, SAPCARArchiveFile, SAPCARInvalidFileException, SAPCARInvalidChecksumException + class PySAPCARCLITest(unittest.TestCase): def setUp(self): + self.mock_file = mock.Mock( + spec=SAPCARArchiveFile, + filename=path.join("test", "bl\x00ah"), + is_directory=lambda: False, + is_file=lambda: True, + perm_mode="-rw-rw-r--", + timestamp_raw=7 + ) + self.mock_dir = mock.Mock( + spec=SAPCARArchiveFile, + filename=path.basename(self.mock_file.filename), + is_directory=lambda: True, + is_file=lambda: False, + perm_mode="-rwxrw-r--", + timestamp_raw=8 + ) + self.mock_archive = mock.Mock( + spec=SAPCARArchive, + files={ + self.mock_file.filename: self.mock_file, + self.mock_dir.filename: self.mock_dir + } + ) self.cli = PySAPCAR() self.cli.mode = "r" self.cli.archive_fd = open(data_filename("car201_test_string.sar"), "rb") @@ -149,6 +175,194 @@ def test_append_with_renaming(self): # For some reason mock_sar.assert_has_calls(calls) fails, even though this passes... self.assertEqual(calls, mock_sar.mock_calls) + def test_extract_open_fails(self): + with mock.patch.object(self.cli, "open_archive", return_value=None): + self.assertIsNone(self.cli.extract(None, None)) + self.log.check() # Check that there's no log messages + + def test_extract_empty_archive(self): + with mock.patch.object(self.cli, "open_archive", return_value=self.mock_archive): + with mock.patch.object(self.cli, "target_files", return_value=[]): + self.cli.extract(None, None) + self.log.check(("pysapcar", "INFO", "pysapcar: 0 file(s) processed")) + + def test_extract_invalid_file_type(self): + key = self.mock_file.filename + self.mock_file.is_file = lambda: False + self.mock_file.is_directory = lambda: False + with mock.patch.object(self.cli, "open_archive", return_value=self.mock_archive): + with mock.patch.object(self.cli, "target_files", return_value=[key]): + key = key.replace("\x00", "") + self.cli.extract(mock.MagicMock(outdir=False), None) + self.log.check( + ("pysapcar", "WARNING", "pysapcar: Invalid file type '{}'".format(key)), + ("pysapcar", "INFO", "pysapcar: 0 file(s) processed") + ) + + @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, chmod=mock.DEFAULT, + makedirs=mock.DEFAULT) + def test_extract_dir_exists(self, makedirs, chmod, utime): + makedirs.side_effect = OSError(17, "Unit test error") + key = self.mock_dir.filename + with mock.patch.object(self.cli, "open_archive", return_value=self.mock_archive): + with mock.patch.object(self.cli, "target_files", return_value=[key]): + key = key.replace("\x00", "") + self.cli.extract(mock.MagicMock(outdir=False), None) + makedirs.assert_called_once_with(key) + chmod.assert_not_called() + utime.assert_not_called() + self.log.check( + ("pysapcar", "INFO", "d {}".format(key)), + ("pysapcar", "INFO", "pysapcar: 1 file(s) processed") + ) + + @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, chmod=mock.DEFAULT, + makedirs=mock.DEFAULT) + def test_extract_dir_creation_fails(self, makedirs, chmod, utime): + makedirs.side_effect = OSError(13, "Unit test error") + key = self.mock_dir.filename + with mock.patch.object(self.cli, "open_archive", return_value=self.mock_archive): + with mock.patch.object(self.cli, "target_files", return_value=[key]): + key = key.replace("\x00", "") + self.cli.extract(mock.MagicMock(outdir=False), None) + makedirs.assert_called_once_with(key) + chmod.assert_not_called() + utime.assert_not_called() + self.log.check( + ("pysapcar", "ERROR", "pysapcar: Could not create directory '{}' ([Errno 13] Unit test error)" + .format(key)), + ("pysapcar", "INFO", "pysapcar: Stopping extraction"), + ("pysapcar", "INFO", "pysapcar: 0 file(s) processed") + ) + + @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, chmod=mock.DEFAULT, + makedirs=mock.DEFAULT) + def test_extract_dir_passes(self, makedirs, chmod, utime): + key = self.mock_dir.filename + with mock.patch.object(self.cli, "open_archive", return_value=self.mock_archive): + with mock.patch.object(self.cli, "target_files", return_value=[key]): + key = key.replace("\x00", "") + self.cli.extract(mock.MagicMock(outdir=False), None) + makedirs.assert_called_once_with(key) + chmod.assert_called_once_with(key, self.mock_dir.perm_mode) + utime.assert_called_once_with(key, (self.mock_dir.timestamp_raw, self.mock_dir.timestamp_raw)) + self.log.check( + ("pysapcar", "INFO", "d {}".format(key)), + ("pysapcar", "INFO", "pysapcar: 1 file(s) processed") + ) + + @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, fchmod=mock.DEFAULT, + makedirs=mock.DEFAULT) + def test_extract_file_intermediate_dir_exists(self, makedirs, fchmod, utime): + makedirs.side_effect = OSError(17, "Unit test error") + key = path.join("test", "bl\x00ah") + mock_file = mock.Mock(spec=SAPCARArchiveFile, is_directory=lambda: False, perm_mode="-rw-rw-r--", + timestamp_raw=7) + mock_arch = mock.Mock(spec=SAPCARArchive, files={key: mock_file}) + with mock.patch.object(self.cli, "open_archive", return_value=mock_arch): + with mock.patch.object(self.cli, "target_files", return_value=[key]): + key = key.replace("\x00", "") + with mock.patch("pysap.sapcarcli.open", mock.mock_open()) as mock_open: + mock_open.return_value.fileno.return_value = 1337 # yo dawg... + self.cli.extract(mock.MagicMock(outdir=False), None) + dirname = path.dirname(key) + makedirs.assert_called_once_with(dirname) + fchmod.assert_called_once_with(1337, "-rw-rw-r--") + utime.assert_called_once_with(key, (7, 7)) + self.log.check( + ("pysapcar", "INFO", "d {}".format(key)), + ("pysapcar", "INFO", "pysapcar: 1 file(s) processed") + ) + + @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, fchmod=mock.DEFAULT, + makedirs=mock.DEFAULT) + def test_extract_file_intermediate_dir_creation_fails(self, makedirs, fchmod, utime): + key = self.mock_file.filename + makedirs.side_effect = OSError(13, "Unit test error") + with mock.patch.object(self.cli, "open_archive", return_value=self.mock_archive): + with mock.patch.object(self.cli, "target_files", return_value=[key]): + key = key.replace("\x00", "") + self.cli.extract(mock.MagicMock(outdir=False), None) + dirname = path.dirname(key) + makedirs.assert_called_once_with(dirname) + fchmod.assert_not_called() + utime.assert_not_called() + self.log.check( + ("pysapcar", "ERROR", "pysapcar: Could not create intermediate directory '{}' for '{}' " + "(Unit test error)".format(dirname, key)), + ("pysapcar", "INFO", "pysapcar: Stopping extraction"), + ("pysapcar", "INFO", "pysapcar: 0 file(s) processed") + ) + + @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, fchmod=mock.DEFAULT, + makedirs=mock.DEFAULT) + def test_extract_file_passes(self, makedirs, fchmod, utime): + key = self.mock_file.filename + with mock.patch.object(self.cli, "open_archive", return_value=self.mock_archive): + with mock.patch.object(self.cli, "target_files", return_value=[key]): + key = key.replace("\x00", "") + with mock.patch("pysap.sapcarcli.open", mock.mock_open()) as mock_open: + mock_open.return_value.fileno.return_value = 1337 # yo dawg... + self.cli.extract(mock.MagicMock(outdir=False), None) + dirname = path.dirname(key) + makedirs.assert_called_once_with(dirname) + fchmod.assert_called_once_with(1337, "-rw-rw-r--") + utime.assert_called_once_with(key, (7, 7)) + self.log.check( + ("pysapcar", "INFO", "d {}".format(dirname)), + ("pysapcar", "INFO", "d {}".format(key)), + ("pysapcar", "INFO", "pysapcar: 1 file(s) processed") + ) + + @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, fchmod=mock.DEFAULT) + def test_extract_file_invalid_file(self, fchmod, utime): + key = self.mock_file.filename + self.mock_file.open.side_effect = SAPCARInvalidFileException("Unit test error") + with mock.patch.object(self.cli, "open_archive", return_value=self.mock_archive): + with mock.patch.object(self.cli, "target_files", return_value=[key]): + self.cli.extract(mock.MagicMock(outdir=False, break_on_error=False), None) + fchmod.assert_not_called() + utime.assert_not_called() + self.log.check( + ("pysapcar", "ERROR", "pysapcar: Invalid SAP CAR file '{}' (Unit test error)" + .format(self.cli.archive_fd.name)), + ("pysapcar", "INFO", "pysapcar: Skipping extraction of file '{}'".format(key)), + ("pysapcar", "INFO", "pysapcar: 0 file(s) processed") + ) + + @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, fchmod=mock.DEFAULT) + def test_extract_file_invalid_checksum(self, fchmod, utime): + key = self.mock_file.filename + self.mock_file.open.side_effect = SAPCARInvalidChecksumException("Unit test error") + with mock.patch.object(self.cli, "open_archive", return_value=self.mock_archive): + with mock.patch.object(self.cli, "target_files", return_value=[key]): + self.cli.extract(mock.MagicMock(outdir=False, break_on_error=False), None) + fchmod.assert_not_called() + utime.assert_not_called() + self.log.check( + ("pysapcar", "ERROR", "pysapcar: Invalid checksum found for file '{}'" + .format(key)), + ("pysapcar", "INFO", "pysapcar: Stopping extraction"), + ("pysapcar", "INFO", "pysapcar: 0 file(s) processed") + ) + + @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, fchmod=mock.DEFAULT) + def test_extract_file_extraction_fails(self, fchmod, utime): + key = self.mock_file.filename + with mock.patch.object(self.cli, "open_archive", return_value=self.mock_archive): + with mock.patch.object(self.cli, "target_files", return_value=[key]): + key = key.replace("\x00", "") + with mock.patch("pysap.sapcarcli.open", mock.mock_open()) as mock_open: + mock_open.side_effect = OSError(13, "Unit test error") + self.cli.extract(mock.MagicMock(outdir=False), None) + fchmod.assert_not_called() + utime.assert_not_called() + self.log.check( + ("pysapcar", "ERROR", "pysapcar: Failed to extract file '{}', (Unit test error)".format(key)), + ("pysapcar", "INFO", "pysapcar: Stopping extraction"), + ("pysapcar", "INFO", "pysapcar: 0 file(s) processed") + ) + def test_suite(): loader = unittest.TestLoader() suite = unittest.TestSuite() From adc4fc218390f7ce7978c963fc35f06501db4c0c Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Thu, 1 Nov 2018 01:15:42 -0300 Subject: [PATCH 39/74] Revert back travis changes --- .travis.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index c180a2a..ff28a0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,19 +61,21 @@ before_script: brew install https://raw.githubusercontent.com/secdev/scapy/master/.travis/pylibpcap.rb; brew install openssl pandoc; fi + +install: # Determine the pip vesion to use - PIP=`which pip || (python --version 2>&1 | grep -q 'Python 2' && which pip2) || (python --version 2>&1 | grep -q 'Python 3' && which pip3)` + - ${PIP} --version + - ${PIP} install . + - ${PIP} install flake8 six + - ${PIP} install -r requirements-docs.txt + - ${PIP} install -r requirements-optional.txt # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics + # XXX: Disabled stopping the build while working on porting to Python2/3 + - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exit-zero # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics -install: - - $PIP install . - - $PIP install flake8 six - - $PIP install -r requirements-docs.txt - - $PIP install -r requirements-optional.txt - script: - python setup.py test - python setup.py doc From c5f82a9006ae9cedb18656eed2b4f09611a62502 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Mon, 22 Jul 2019 14:43:03 -0700 Subject: [PATCH 40/74] Matching environments supported by Scapy --- .appveyor.yml | 8 ++++---- .travis.yml | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 2fcbb07..98ee068 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -16,13 +16,13 @@ environment: PYTHON_ARCH: "64" PLAT_NAME: "win-amd64" - - PYTHON: "C:\\Python27" - PYTHON_VERSION: "3.6.x" + - PYTHON: "C:\\Python37" + PYTHON_VERSION: "3.7.x" ARCH: "32" PLAT_NAME: "win32" - - PYTHON: "C:\\Python27-x64" - PYTHON_VERSION: "3.6.x" + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7.x" ARCH: "64" PLAT_NAME: "win-amd64" diff --git a/.travis.yml b/.travis.yml index fc7b77f..66d06da 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,12 @@ matrix: - CXX=clang++ CC=clang - os: linux - python: 3.6 + python: 3.7 + env: + - CXX=clang++ CC=clang + + - os: linux + python: 3.7 env: - CXX=g++ CC=gcc From ce9854dd812898ff5c51d245f8e66ed3dfb97fe2 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Mon, 22 Jul 2019 17:30:18 -0700 Subject: [PATCH 41/74] Bumping six version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 430004e..c418f67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -six==1.11.0 +six==1.12 scapy==2.4.3rc3 From 9c456557929626a6467caaa1e2d324b88d08ec43 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Mon, 22 Jul 2019 17:38:21 -0700 Subject: [PATCH 42/74] Fixing socketserver imports --- pysap/SAPNI.py | 2 +- tests/test_sapni.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pysap/SAPNI.py b/pysap/SAPNI.py index 9f896df..c4352a5 100644 --- a/pysap/SAPNI.py +++ b/pysap/SAPNI.py @@ -23,7 +23,7 @@ from select import select from struct import unpack from threading import Event -from SocketServer import BaseRequestHandler, ThreadingMixIn, TCPServer +from six.moves.socketserver import BaseRequestHandler, ThreadingMixIn, TCPServer # External imports from scapy.fields import LenField from scapy.packet import Packet, Raw diff --git a/tests/test_sapni.py b/tests/test_sapni.py index 96f98e9..e8968c4 100755 --- a/tests/test_sapni.py +++ b/tests/test_sapni.py @@ -22,7 +22,7 @@ import unittest from threading import Thread from struct import pack, unpack -from SocketServer import BaseRequestHandler, ThreadingTCPServer +from six.moves.socketserver import BaseRequestHandler, ThreadingTCPServer # External imports from scapy.fields import StrField from scapy.packet import Packet, Raw From 13d42281f2414e23ce1c9a536655332a292ecb8d Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Mon, 22 Jul 2019 18:54:19 -0700 Subject: [PATCH 43/74] Strings to bytes on SAP NI layer --- pysap/SAPNI.py | 2 +- tests/test_sapni.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pysap/SAPNI.py b/pysap/SAPNI.py index c4352a5..6413338 100644 --- a/pysap/SAPNI.py +++ b/pysap/SAPNI.py @@ -124,7 +124,7 @@ def recv(self): log_sapni.debug("To receive %d bytes", nilength) # Receive the whole NI packet (length+payload) - nidata = '' + nidata = b'' while len(nidata) < nilength + 4: nidata += self.ins.recv(nilength - len(nidata) + 4) if len(nidata) == 0: diff --git a/tests/test_sapni.py b/tests/test_sapni.py index e8968c4..4cd3bc1 100755 --- a/tests/test_sapni.py +++ b/tests/test_sapni.py @@ -33,13 +33,13 @@ class PySAPNITest(unittest.TestCase): - test_string = "LALA" * 10 + test_string = b"LALA" * 10 def test_sapni_building(self): """Test SAPNI length field building""" sapni = SAPNI() / self.test_string - (sapni_length, ) = unpack("!I", str(sapni)[:4]) + (sapni_length, ) = unpack("!I", bytes(sapni)[:4]) self.assertEqual(sapni_length, len(self.test_string)) self.assertEqual(sapni.payload.load, self.test_string) @@ -70,7 +70,7 @@ class SAPNITestHandlerKeepAlive(SAPNITestHandler): """Basic SAP NI keep alive server""" def handle(self): - self.request.sendall("\x00\x00\x00\x08NI_PING\x00") + self.request.sendall(b"\x00\x00\x00\x08NI_PING\x00") SAPNITestHandler.handle(self) @@ -78,14 +78,14 @@ class SAPNITestHandlerClose(SAPNITestHandler): """Basic SAP NI server that closes the connection""" def handle(self): - self.request.send("") + self.request.send(b"") class PySAPNIStreamSocketTest(unittest.TestCase): test_port = 8005 test_address = "127.0.0.1" - test_string = "TEST" * 10 + test_string = b"TEST" * 10 def start_server(self, handler_cls): self.server = ThreadingTCPServer((self.test_address, self.test_port), @@ -223,7 +223,7 @@ class PySAPNIServerTest(unittest.TestCase): test_port = 8005 test_address = "127.0.0.1" - test_string = "TEST" * 10 + test_string = b"TEST" * 10 handler_cls = SAPNIServerTestHandler def setUp(self): @@ -261,7 +261,7 @@ class PySAPNIProxyTest(unittest.TestCase): test_proxyport = 8005 test_serverport = 8006 test_address = "127.0.0.1" - test_string = "TEST" * 10 + test_string = b"TEST" * 10 proxyhandler_cls = SAPNIProxyHandler serverhandler_cls = SAPNIServerTestHandler @@ -327,7 +327,7 @@ def process_server(self, packet): response = sock.recv(1024) - expected_reponse = self.test_string + "Client" + "Server" + expected_reponse = self.test_string + b"Client" + b"Server" self.assertEqual(len(response), len(expected_reponse) + 8) self.assertEqual(unpack("!I", response[:4]), (len(expected_reponse) + 4, )) self.assertEqual(unpack("!I", response[4:8]), (len(self.test_string) + 6, )) From 8a02d6bf3d0152858bb5893b287089858938ae78 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Tue, 5 Nov 2019 15:41:32 -0800 Subject: [PATCH 44/74] Adding Python 3.8 to Travis --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.travis.yml b/.travis.yml index f72e039..21575fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,16 @@ matrix: env: - CXX=g++ CC=gcc + - os: linux + python: 3.8 + env: + - CXX=clang++ CC=clang + + - os: linux + python: 3.8 + env: + - CXX=g++ CC=gcc + - os: osx language: generic From 6a92be825ab7216adf820f32c44d2388d25cde85 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Tue, 5 Nov 2019 15:48:08 -0800 Subject: [PATCH 45/74] Using six for queue --- pysap/utils/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysap/utils/__init__.py b/pysap/utils/__init__.py index e6d297f..e71fb6a 100644 --- a/pysap/utils/__init__.py +++ b/pysap/utils/__init__.py @@ -19,7 +19,7 @@ # Standard imports -from Queue import Queue +from six.moves import queue from threading import Thread, Event @@ -67,7 +67,7 @@ def run(self): class ThreadPool(object): """Pool of threads consuming tasks from a queue""" def __init__(self, num_threads): - self.tasks = Queue(num_threads) + self.tasks = queue(num_threads) for _ in range(num_threads): WorkerQueue(self.tasks) From 688425aa02ce6cef23116b7ffe0541e18422653f Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Thu, 13 Feb 2020 15:19:11 -0800 Subject: [PATCH 46/74] Moving compat helpers to utils to avoid import issues --- pysap/SAPCAR.py | 2 +- pysap/utils/__init__.py | 19 +++++++++++++++++++ pysap/utils/six.py | 39 --------------------------------------- 3 files changed, 20 insertions(+), 40 deletions(-) delete mode 100644 pysap/utils/six.py diff --git a/pysap/SAPCAR.py b/pysap/SAPCAR.py index 637e0e6..39a275b 100644 --- a/pysap/SAPCAR.py +++ b/pysap/SAPCAR.py @@ -32,8 +32,8 @@ PacketField, StrFixedLenField, PacketListField, ConditionalField, LESignedIntField, StrField, LELongField) # Custom imports +from pysap.utils import unicode from pysap.utils.fields import (PacketNoPadded, StrNullFixedLenField, PacketListStopField) -from pysap.utils.six import unicode from pysapcompress import (decompress, compress, ALG_LZH, CompressError, DecompressError) diff --git a/pysap/utils/__init__.py b/pysap/utils/__init__.py index 1c21959..d4dd1cf 100644 --- a/pysap/utils/__init__.py +++ b/pysap/utils/__init__.py @@ -21,6 +21,7 @@ # Standard imports from six.moves import queue from threading import Thread, Event +from six import binary_type, text_type class Worker(Thread): @@ -78,3 +79,21 @@ def add_task(self, func, *args, **kargs): def wait_completion(self): """Wait for completion of all the tasks in the queue""" self.tasks.join() + + +# All custom general purpose Python 2/3 compatibility should go here + + +def unicode(string): + """Convert given string to unicode string + + :param string: String to convert + :type string: bytes | str | unicode + :return: six.text_type + """ + string_type = type(string) + if string_type == binary_type: + return string.decode() + elif string_type == text_type: + return string + raise ValueError("Expected bytes or str, got {}".format(string_type)) \ No newline at end of file diff --git a/pysap/utils/six.py b/pysap/utils/six.py deleted file mode 100644 index 65d47fa..0000000 --- a/pysap/utils/six.py +++ /dev/null @@ -1,39 +0,0 @@ -# =========== -# pysap - Python library for crafting SAP's network protocols packets -# -# Copyright (C) 2012-2018 by Martin Gallo, Core Security -# -# The library was designed and developed by Martin Gallo from -# Core Security's CoreLabs team. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# ============== - -# All custom general purpose Python 2/3 compatibility should go here - -from __future__ import absolute_import - -import six - - -def unicode(string): - """ - Convert given string to unicode string - :param string: String to convert - :type string: bytes | str | unicode - :return: six.text_type - """ - string_type = type(string) - if string_type == six.binary_type: - return string.decode() - elif string_type == six.text_type: - return string - raise ValueError("Expected bytes or str, got {}".format(string_type)) From 41c2ffbc9c2ae01fafc978d5441e5a4c50084328 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Thu, 13 Feb 2020 15:20:50 -0800 Subject: [PATCH 47/74] Execution bit --- tests/test_sapcarcli.py | 0 tests/test_sapcredv2.py | 0 tests/test_sappse.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/test_sapcarcli.py mode change 100644 => 100755 tests/test_sapcredv2.py mode change 100644 => 100755 tests/test_sappse.py diff --git a/tests/test_sapcarcli.py b/tests/test_sapcarcli.py old mode 100644 new mode 100755 diff --git a/tests/test_sapcredv2.py b/tests/test_sapcredv2.py old mode 100644 new mode 100755 diff --git a/tests/test_sappse.py b/tests/test_sappse.py old mode 100644 new mode 100755 From 39d46e570764ab53b29b662da91097b48625b6f2 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Thu, 13 Feb 2020 15:21:01 -0800 Subject: [PATCH 48/74] Fixed test suite loading --- setup.py | 2 +- tests/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a7681a2..2b67e4f 100755 --- a/setup.py +++ b/setup.py @@ -121,7 +121,7 @@ def run(self): scripts=['bin/pysapcar', 'bin/pysapgenpse'], # Tests command - test_suite='tests', + test_suite='tests.test_suite', # Documentation commands cmdclass={'doc': DocumentationCommand, diff --git a/tests/__init__.py b/tests/__init__.py index 57e8047..984c2b4 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -22,7 +22,7 @@ import unittest -test_suite = unittest.defaultTestLoader.discover('.', '*_test.py') +test_suite = unittest.defaultTestLoader.discover('.', 'test_*.py') if __name__ == '__main__': From 30e9f014f7d82e0d3d127252a36d638d5ad2567d Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Thu, 13 Feb 2020 15:31:21 -0800 Subject: [PATCH 49/74] Using bytes for NI_PING and NI_PONG constants, fixes keep alive --- pysap/SAPNI.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pysap/SAPNI.py b/pysap/SAPNI.py index d058a81..0e7b761 100644 --- a/pysap/SAPNI.py +++ b/pysap/SAPNI.py @@ -59,13 +59,13 @@ class SAPNI(Packet): fields_desc = [LenField("length", None, fmt="!I")] # Constants for keep-alive messages - SAPNI_PING = "NI_PING\x00" + SAPNI_PING = b"NI_PING\x00" """ :cvar: Constant for keep-alive request messages (NI_PING) - :type: C{string} """ + :type: C{bytes} """ - SAPNI_PONG = "NI_PONG\x00" + SAPNI_PONG = b"NI_PONG\x00" """ :cvar: Constant for keep-alive response messages (NI_PONG) - :type: C{string} """ + :type: C{bytes} """ class SAPNIStreamSocket(StreamSocket): From b40ed917654bc191c0ace2a8707d191570cd9f5a Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Thu, 13 Feb 2020 15:53:36 -0800 Subject: [PATCH 50/74] Using bytes across some packets and tests --- pysap/SAPRouter.py | 33 +++++++++++++++++---------------- tests/test_saprouter.py | 3 ++- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pysap/SAPRouter.py b/pysap/SAPRouter.py index 9156856..685b0f1 100644 --- a/pysap/SAPRouter.py +++ b/pysap/SAPRouter.py @@ -34,6 +34,7 @@ from pysap.SAPSNC import SAPSNCFrame from pysap.SAPNI import (SAPNI, SAPNIStreamSocket, SAPNIProxy, SAPNIProxyHandler) +from pysap.utils import unicode from pysap.utils.fields import (PacketNoPadded, StrNullFixedLenField) @@ -182,11 +183,11 @@ def from_hops(cls, route_hops): """ result = "" for route_hop in route_hops: - result += "/H/{}".format(route_hop.hostname) + result += "/H/{}".format(unicode(route_hop.hostname)) if route_hop.port: - result += "/S/{}".format(route_hop.port) + result += "/S/{}".format(unicode(route_hop.port)) if route_hop.password: - result += "/W/{}".format(route_hop.password) + result += "/W/{}".format(unicode(route_hop.password)) return result @@ -392,25 +393,25 @@ class SAPRouter(Packet): SAPROUTER_DEFAULT_VERSION = 40 # Constants for router types - SAPROUTER_ROUTE = "NI_ROUTE" + SAPROUTER_ROUTE = b"NI_ROUTE" """ :cvar: Constant for route packets - :type: C{string} """ + :type: C{bytes} """ - SAPROUTER_ADMIN = "ROUTER_ADM" + SAPROUTER_ADMIN = b"ROUTER_ADM" """ :cvar: Constant for administration packets - :type: C{string} """ + :type: C{bytes} """ - SAPROUTER_ERROR = "NI_RTERR" + SAPROUTER_ERROR = b"NI_RTERR" """ :cvar: Constant for error information packets - :type: C{string} """ + :type: C{bytes} """ - SAPROUTER_CONTROL = "NI_RTERR" + SAPROUTER_CONTROL = b"NI_RTERR" """ :cvar: Constant for control messages packets - :type: C{string} """ + :type: C{bytes} """ - SAPROUTER_PONG = "NI_PONG" + SAPROUTER_PONG = b"NI_PONG" """ :cvar: Constant for route accepted packets - :type: C{string} """ + :type: C{bytes} """ router_type_values = [ SAPROUTER_ADMIN, @@ -420,7 +421,7 @@ class SAPRouter(Packet): SAPROUTER_PONG, ] """ :cvar: List of known packet types - :type: ``list`` of C{string} """ + :type: ``list`` of C{bytes} """ name = "SAP Router" fields_desc = [ @@ -445,7 +446,7 @@ class SAPRouter(Packet): ConditionalField(ShortField("adm_unused", 0x00), lambda pkt:router_is_admin(pkt) and pkt.adm_command not in [10, 11, 12, 13]), # Info Request fields - ConditionalField(StrNullFixedLenField("adm_password", "", 19), lambda pkt:router_is_admin(pkt) and pkt.adm_command in [2]), + ConditionalField(StrNullFixedLenField("adm_password", b"", 19), lambda pkt:router_is_admin(pkt) and pkt.adm_command in [2]), # Cancel Route fields ConditionalField(FieldLenField("adm_client_count", None, count_of="adm_client_ids", fmt="H"), lambda pkt:router_is_admin(pkt) and pkt.adm_command in [6]), @@ -470,7 +471,7 @@ class SAPRouter(Packet): # Control Message fields ConditionalField(IntField("control_text_length", 0), lambda pkt: router_is_control(pkt) and pkt.opcode != 0), - ConditionalField(StrField("control_text_value", "*ERR"), lambda pkt: router_is_control(pkt) and pkt.opcode != 0), + ConditionalField(StrField("control_text_value", b"*ERR"), lambda pkt: router_is_control(pkt) and pkt.opcode != 0), # SNC Frame fields ConditionalField(PacketField("snc_frame", None, SAPSNCFrame), lambda pkt: router_is_control(pkt) and pkt.opcode in [70, 71]) diff --git a/tests/test_saprouter.py b/tests/test_saprouter.py index c51c2f0..9ce0372 100755 --- a/tests/test_saprouter.py +++ b/tests/test_saprouter.py @@ -105,6 +105,7 @@ def handle_data(self): self.packet.decode_payload_as(SAPRouter) route_request = self.packet[SAPRouter] + route_request.show() if router_is_route(route_request): if route_request.route_string[1].hostname == "10.0.0.1" and \ @@ -218,7 +219,7 @@ def test_saproutedstreamsocket_getnisocket(self): route = [SAPRouterRouteHop(hostname=self.test_address, port=self.test_port)] self.client = SAPRoutedStreamSocket.get_nisocket("10.0.0.1", - "3200", + 3200, route=route, router_version=40) From d66d39dfa3f21b69f9c2fbd07a691504ac58ec89 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Sun, 8 Nov 2020 15:14:38 -0800 Subject: [PATCH 51/74] Moving to 2.x branch, Python 2/3 version --- ChangeLog.md | 3 ++- README.md | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 23b9c9f..a0151be 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -2,9 +2,10 @@ Changelog ========= -v0.1.19 - 2020-XX-XX +v0.2.1 - 2020-XX-XX -------------------- +- Python 2/3 compatible codebase - Using Scapy version 2.4.4. - `requirements-examples.txt`: Renamed to match `setup.py`'s extra. - `pysap/SAPHDB.py`: Implementation of GSS-based auth method with Kerberos 5. diff --git a/README.md b/README.md index 2313cc7..9621a53 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ pysap - Python library for crafting SAP's network protocols packets SECUREAUTH LABS. Copyright (C) 2020 SecureAuth Corporation. All rights reserved. -Version 0.1.19.dev0 (XXX 2020) +Version 0.2.1.dev0 (XXX 2020) Overview @@ -21,7 +21,7 @@ formats as well. While some of them are standard and well-known protocols, other are proprietaries and public information is generally not available. [pysap](https://www.secureauth.com/labs/open-source-tools/pysap) -is an open source Python 2 library that provides modules for crafting and sending packets +is an open source Python 2/3 library that provides modules for crafting and sending packets using SAP's `NI`, `Diag`, `Enqueue`, `Router`, `MS`, `SNC`, `IGS`, `RFC` and `HDB` protocols. In addition, support for creating and parsing different proprietary file formats is included. The modules are built on top of [Scapy](https://scapy.net/) and are @@ -66,9 +66,7 @@ To install pysap simply run: $ pip install pysap -pysap is compatible and tested with Python 2.7. A Python 2/3 compatible version -is [in the workings](https://github.com/SecureAuthCorp/pysap/tree/python2-3) but -it's [not finalized yet](https://github.com/SecureAuthCorp/pysap/projects/1). +pysap is compatible and tested with Python 2.7/3.8. Documentation ------------- From 1adde99e8eb26c1f0c9f47a63ffc8a5eaa69bcef Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Sun, 8 Nov 2020 15:22:52 -0800 Subject: [PATCH 52/74] CI: Added Python 3 versions to GH Actions --- .github/workflows/build_and_test.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index cc799de..ad58358 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7] + python-version: [2.7, 3.6, 3.7, 3.8] steps: - name: Checkout pysap uses: actions/checkout@v2 @@ -21,7 +21,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python2 -m pip install --upgrade pip wheel + python -m pip install --upgrade pip wheel pip install flake8 six pip install -r requirements.txt - name: Run flake8 tests @@ -37,12 +37,15 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - python-version: [2.7] + python-version: [2.7, 3.6, 3.7, 3.8] experimental: [false] include: - os: windows-latest python-version: 2.7 experimental: true + - os: windows-latest + python-version: 3.8 + experimental: true steps: - name: Checkout pysap uses: actions/checkout@v2 @@ -83,7 +86,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7] + python-version: [2.7, 3.6, 3.7, 3.8] steps: - name: Checkout pysap uses: actions/checkout@v2 @@ -96,22 +99,22 @@ jobs: sudo apt-get install pandoc texlive-latex-base - name: Install Python dependencies run: | - python2 -m pip install --upgrade pip wheel + python -m pip install --upgrade pip wheel pip install -r requirements-docs.txt - name: Install the library run: | - python2 setup.py install + python setup.py install - name: Pre-run documentation notebooks run: | - python2 setup.py notebooks + python setup.py notebooks - name: Build source artifact run: | - python2 setup.py sdist + python setup.py sdist - name: Build documentation run: | - python2 setup.py doc + python setup.py doc - name: Upload source artifact uses: actions/upload-artifact@v2 From 10d0ff9888548b211e76ba6e9e2b6d02160f0028 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Tue, 10 Nov 2020 16:26:29 -0800 Subject: [PATCH 53/74] Adding crypto and HDB tests, passing them on Python3 --- pysap/utils/crypto.py | 8 +++++--- tests/{crypto_test.py => test_crypto.py} | 0 tests/{saphdb_test.py => test_saphdb.py} | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) rename tests/{crypto_test.py => test_crypto.py} (100%) rename tests/{saphdb_test.py => test_saphdb.py} (96%) diff --git a/pysap/utils/crypto.py b/pysap/utils/crypto.py index d7db664..b5a53b9 100644 --- a/pysap/utils/crypto.py +++ b/pysap/utils/crypto.py @@ -21,6 +21,8 @@ import os import math # External imports +from six import b +from six.moves import xrange from cryptography.exceptions import InvalidKey from cryptography.hazmat.primitives.hmac import HMAC from cryptography.hazmat.primitives.ciphers import Cipher @@ -139,7 +141,7 @@ def digest(inp): def to_int(value): if value == b'': return 0 - return long(value.encode("hex"), 16) + return int(value.encode("hex"), 16) def to_bytes(value): value = "%x" % value @@ -272,7 +274,7 @@ def get_client_key(self): return os.urandom(self.CLIENT_KEY_SIZE) def salt_key(self, password, salt, rounds): - hmac = HMAC(password, self.ALGORITHM(), self.backend) + hmac = HMAC(b(password), self.ALGORITHM(), self.backend) hmac.update(salt) return hmac.finalize() @@ -325,4 +327,4 @@ class SCRAM_PBKDF2SHA256(SCRAM_SHA256): def salt_key(self, password, salt, rounds): pbkdf2 = PBKDF2HMAC(self.ALGORITHM(), self.CLIENT_PROOF_SIZE, salt, rounds, self.backend) - return pbkdf2.derive(password) + return pbkdf2.derive(b(password)) diff --git a/tests/crypto_test.py b/tests/test_crypto.py similarity index 100% rename from tests/crypto_test.py rename to tests/test_crypto.py diff --git a/tests/saphdb_test.py b/tests/test_saphdb.py similarity index 96% rename from tests/saphdb_test.py rename to tests/test_saphdb.py index 00fc8c5..3d28651 100755 --- a/tests/saphdb_test.py +++ b/tests/test_saphdb.py @@ -21,7 +21,8 @@ import sys import unittest from threading import Thread -from SocketServer import BaseRequestHandler, ThreadingTCPServer +# External imports +from six.moves.socketserver import BaseRequestHandler, ThreadingTCPServer # Custom imports from pysap.SAPHDB import SAPHDBConnection From 8595e4c99f6ebef1ac45453df907298c8885ac93 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Tue, 10 Nov 2020 16:53:43 -0800 Subject: [PATCH 54/74] Python2/3: Fixing exception accessors --- bin/pysapgenpse | 6 +++--- examples/hdb_auth.py | 4 ++-- examples/hdb_discovery.py | 2 +- pysap/SAPNI.py | 3 ++- pysap/sapcarcli.py | 8 ++++---- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/bin/pysapgenpse b/bin/pysapgenpse index eed6572..a758b94 100755 --- a/bin/pysapgenpse +++ b/bin/pysapgenpse @@ -145,7 +145,7 @@ class PySAPGenPSE(object): obj = cls(f.read()) self.logger.info("pysapgenpse: Reading {} file '{}'".format(type, filename)) except IOError as e: - self.logger.error("pysapgenpse: Error reading {} file '{}' ({})".format(type, filename, e.message)) + self.logger.error("pysapgenpse: Error reading {} file '{}' ({})".format(type, filename, str(e))) return None return obj @@ -231,7 +231,7 @@ class PySAPGenPSE(object): self.decrypt_cred(cred, options.username, options.output, options.decrypt_provider) except Exception as e: self.logger.error("pysapgenpse: Error trying to decrypt with username '{}'".format(options.username)) - self.logger.error(e) + self.logger.error(str(e)) i += 1 @@ -260,7 +260,7 @@ class PySAPGenPSE(object): pin = plain.decrypt_provider(cred) except Exception as e: self.logger.error("pysapgenpse: Unable to decrypt using the provider {} ({}), writing plain blob".format( - plain.option1.val, e.message)) + plain.option1.val, str(e))) pin = plain.pin.val else: pin = plain.pin.val diff --git a/examples/hdb_auth.py b/examples/hdb_auth.py index eaba177..a5dee17 100755 --- a/examples/hdb_auth.py +++ b/examples/hdb_auth.py @@ -204,9 +204,9 @@ def main(): logging.debug("[*] Connection with HANA database server closed") except SAPHDBAuthenticationError as e: - logging.error("[-] Authentication error: %s" % e.message) + logging.error("[-] Authentication error: %s" % str(e)) except SAPHDBConnectionError as e: - logging.error("[-] Connection error: %s" % e.message) + logging.error("[-] Connection error: %s" % str(e)) except KeyboardInterrupt: logging.info("[-] Connection canceled") diff --git a/examples/hdb_discovery.py b/examples/hdb_discovery.py index a49bbe5..504e29f 100755 --- a/examples/hdb_discovery.py +++ b/examples/hdb_discovery.py @@ -163,7 +163,7 @@ def main(): except SocketError: logging.error("[-] Tenant '%s' doesn't exist" % tenant) except SAPHDBConnectionError as e: - logging.error("[-] Connection error: %s" % e.message) + logging.error("[-] Connection error: %s" % str(e)) except KeyboardInterrupt: logging.info("[-] Connection canceled") diff --git a/pysap/SAPNI.py b/pysap/SAPNI.py index f4a793b..662da26 100644 --- a/pysap/SAPNI.py +++ b/pysap/SAPNI.py @@ -473,8 +473,9 @@ def handle(self): self.handle_data() except socket.error as e: + errno, message = e.args log_sapni.debug("SAPNIServerHandler: Error handling data or client %s disconnected, %s (errno %d)", - self.client_address, e.message, e.errno) + self.client_address, message, errno) break def handle_data(self): diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index 4ef12d1..cec8c64 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -130,7 +130,7 @@ def main(self): try: self.archive_fd = open(options.filename, self.mode) except IOError as e: - self.logger.error("pysapcar: error opening '%s' (%s)" % (options.filename, e.strerror)) + self.logger.error("pysapcar: error opening '%s' (%s)" % (options.filename, str(e)) return else: self.archive_fd = stdin @@ -154,7 +154,7 @@ def open_archive(self): sapcar = SAPCARArchive(self.archive_fd, mode=self.mode) self.logger.info("pysapcar: Processing archive '%s' (version %s)", self.archive_fd.name, sapcar.version) except Exception as e: - self.logger.error("pysapcar: Error processing archive '%s' (%s)", self.archive_fd.name, e.message) + self.logger.error("pysapcar: Error processing archive '%s' (%s)", self.archive_fd.name, str(e)) return None return sapcar @@ -288,7 +288,7 @@ def extract(self, options, args): try: data = fil.open(enforce_checksum=options.enforce_checksum).read() except (SAPCARInvalidFileException, DecompressError) as e: - self.logger.error("pysapcar: Invalid SAP CAR file '%s' (%s)", self.archive_fd.name, e.message) + self.logger.error("pysapcar: Invalid SAP CAR file '%s' (%s)", self.archive_fd.name, str(e)) if options.break_on_error: flag = STOP else: @@ -317,7 +317,7 @@ def extract(self, options, args): # Set the timestamp utime(filename, (fil.timestamp_raw, fil.timestamp_raw)) except (IOError, OSError) as e: - self.logger.error("pysapcar: Failed to extract file '%s', (%s)", filename, e.strerror) + self.logger.error("pysapcar: Failed to extract file '%s', (%s)", filename, str(e)) if options.break_on_error: self.logger.info("pysapcar: Stopping extraction") break From 953af111fc72defbc9111c130c1ec1ff6f4970f8 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Tue, 10 Nov 2020 17:56:30 -0800 Subject: [PATCH 55/74] Python2/3: Fixing pysapcarcli unit tests --- pysap/sapcarcli.py | 6 ++-- tests/test_sapcarcli.py | 67 +++++++++++++++++++++-------------------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/pysap/sapcarcli.py b/pysap/sapcarcli.py index cec8c64..8da5c84 100644 --- a/pysap/sapcarcli.py +++ b/pysap/sapcarcli.py @@ -62,7 +62,7 @@ def __init__(self): # Instance attributes self.mode = None - self.log_level = 0 + self.verbose = False self.archive_fd = None @staticmethod @@ -130,7 +130,7 @@ def main(self): try: self.archive_fd = open(options.filename, self.mode) except IOError as e: - self.logger.error("pysapcar: error opening '%s' (%s)" % (options.filename, str(e)) + self.logger.error("pysapcar: error opening '%s' (%s)" % (options.filename, str(e))) return else: self.archive_fd = stdin @@ -154,7 +154,7 @@ def open_archive(self): sapcar = SAPCARArchive(self.archive_fd, mode=self.mode) self.logger.info("pysapcar: Processing archive '%s' (version %s)", self.archive_fd.name, sapcar.version) except Exception as e: - self.logger.error("pysapcar: Error processing archive '%s' (%s)", self.archive_fd.name, str(e)) + self.logger.error("pysapcar: Error processing archive '%s' (%s)", self.archive_fd, str(e)) return None return sapcar diff --git a/tests/test_sapcarcli.py b/tests/test_sapcarcli.py index d697b16..50e1cce 100755 --- a/tests/test_sapcarcli.py +++ b/tests/test_sapcarcli.py @@ -125,12 +125,12 @@ def test_list_succeeds(self): self.cli.list(None, None) messages = ("{} {:>10} {} {}".format(fil.permissions, fil.size, fil.timestamp, fil.filename) for fil in archive_attrs["files"].values()) - logs = (("pysapcar", "INFO", message) for message in messages) + logs = (("pysap.pysapcar", "INFO", message) for message in messages) self.log.check_present(*logs, order_matters=False) def test_append_no_args(self): self.cli.append(None, []) - self.log.check_present(("pysapcar", "ERROR", "pysapcar: no files specified for appending")) + self.log.check_present(("pysap.pysapcar", "ERROR", "pysapcar: no files specified for appending")) def test_append_open_fails(self): with mock.patch.object(self.cli, "open_archive", return_value=None): @@ -146,8 +146,8 @@ def test_append_no_renaming(self): # it's pretty in it's obscurity. And because someone (probably future me) will probably bang one's head # against the same wall I just did before coming to the same realization I just did, so here's a little # something for the next poor soul. May it make your head-against-the-wall session shorter than mine was. - # debugs = (("pysapcar", "DEBUG", str(names[i + 1:])) for i in range(len(names))) - infos = (("pysapcar", "INFO", "d {}".format(name)) for name in names) + # debugs = (("pysap.pysapcar", "DEBUG", str(names[i + 1:])) for i in range(len(names))) + infos = (("pysap.pysapcar", "INFO", "d {}".format(name)) for name in names) calls = [mock.call.add_file(name, archive_filename=name) for name in names] calls.append(mock.call.write()) with mock.patch.object(self.cli, "open_archive", return_value=mock_sar): @@ -162,9 +162,9 @@ def test_append_with_renaming(self): names = ["test", "/n", "blah", "test", "test", "/n", "blah2"] archive_names = [("test", "blah"), ("test", "blah2")] # List instead of generator, as we need to insert one line - infos = [("pysapcar", "INFO", "d {} (original name {})".format(archive_name, name)) + infos = [("pysap.pysapcar", "INFO", "d {} (original name {})".format(archive_name, name)) for name, archive_name in archive_names] - infos.insert(1, ("pysapcar", "INFO", "d {}".format("test"))) + infos.insert(1, ("pysap.pysapcar", "INFO", "d {}".format("test"))) mock_sar = mock.Mock(spec=SAPCARArchive) calls = [mock.call.add_file(name, archive_filename=archive_name) for name, archive_name in archive_names] calls.insert(1, mock.call.add_file("test", archive_filename="test")) @@ -184,7 +184,7 @@ def test_extract_empty_archive(self): with mock.patch.object(self.cli, "open_archive", return_value=self.mock_archive): with mock.patch.object(self.cli, "target_files", return_value=[]): self.cli.extract(None, None) - self.log.check(("pysapcar", "INFO", "pysapcar: 0 file(s) processed")) + self.log.check(("pysap.pysapcar", "INFO", "pysapcar: 0 file(s) processed")) def test_extract_invalid_file_type(self): key = self.mock_file.filename @@ -195,8 +195,8 @@ def test_extract_invalid_file_type(self): key = key.replace("\x00", "") self.cli.extract(mock.MagicMock(outdir=False), None) self.log.check( - ("pysapcar", "WARNING", "pysapcar: Invalid file type '{}'".format(key)), - ("pysapcar", "INFO", "pysapcar: 0 file(s) processed") + ("pysap.pysapcar", "WARNING", "pysapcar: Invalid file type '{}'".format(key)), + ("pysap.pysapcar", "INFO", "pysapcar: 0 file(s) processed") ) @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, chmod=mock.DEFAULT, @@ -212,8 +212,8 @@ def test_extract_dir_exists(self, makedirs, chmod, utime): chmod.assert_not_called() utime.assert_not_called() self.log.check( - ("pysapcar", "INFO", "d {}".format(key)), - ("pysapcar", "INFO", "pysapcar: 1 file(s) processed") + ("pysap.pysapcar", "INFO", "d {}".format(key)), + ("pysap.pysapcar", "INFO", "pysapcar: 1 file(s) processed") ) @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, chmod=mock.DEFAULT, @@ -229,10 +229,10 @@ def test_extract_dir_creation_fails(self, makedirs, chmod, utime): chmod.assert_not_called() utime.assert_not_called() self.log.check( - ("pysapcar", "ERROR", "pysapcar: Could not create directory '{}' ([Errno 13] Unit test error)" + ("pysap.pysapcar", "ERROR", "pysapcar: Could not create directory '{}' ([Errno 13] Unit test error)" .format(key)), - ("pysapcar", "INFO", "pysapcar: Stopping extraction"), - ("pysapcar", "INFO", "pysapcar: 0 file(s) processed") + ("pysap.pysapcar", "INFO", "pysapcar: Stopping extraction"), + ("pysap.pysapcar", "INFO", "pysapcar: 0 file(s) processed") ) @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, chmod=mock.DEFAULT, @@ -247,8 +247,8 @@ def test_extract_dir_passes(self, makedirs, chmod, utime): chmod.assert_called_once_with(key, self.mock_dir.perm_mode) utime.assert_called_once_with(key, (self.mock_dir.timestamp_raw, self.mock_dir.timestamp_raw)) self.log.check( - ("pysapcar", "INFO", "d {}".format(key)), - ("pysapcar", "INFO", "pysapcar: 1 file(s) processed") + ("pysap.pysapcar", "INFO", "d {}".format(key)), + ("pysap.pysapcar", "INFO", "pysapcar: 1 file(s) processed") ) @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, fchmod=mock.DEFAULT, @@ -270,8 +270,8 @@ def test_extract_file_intermediate_dir_exists(self, makedirs, fchmod, utime): fchmod.assert_called_once_with(1337, "-rw-rw-r--") utime.assert_called_once_with(key, (7, 7)) self.log.check( - ("pysapcar", "INFO", "d {}".format(key)), - ("pysapcar", "INFO", "pysapcar: 1 file(s) processed") + ("pysap.pysapcar", "INFO", "d {}".format(key)), + ("pysap.pysapcar", "INFO", "pysapcar: 1 file(s) processed") ) @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, fchmod=mock.DEFAULT, @@ -288,10 +288,10 @@ def test_extract_file_intermediate_dir_creation_fails(self, makedirs, fchmod, ut fchmod.assert_not_called() utime.assert_not_called() self.log.check( - ("pysapcar", "ERROR", "pysapcar: Could not create intermediate directory '{}' for '{}' " + ("pysap.pysapcar", "ERROR", "pysapcar: Could not create intermediate directory '{}' for '{}' " "(Unit test error)".format(dirname, key)), - ("pysapcar", "INFO", "pysapcar: Stopping extraction"), - ("pysapcar", "INFO", "pysapcar: 0 file(s) processed") + ("pysap.pysapcar", "INFO", "pysapcar: Stopping extraction"), + ("pysap.pysapcar", "INFO", "pysapcar: 0 file(s) processed") ) @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, fchmod=mock.DEFAULT, @@ -309,9 +309,9 @@ def test_extract_file_passes(self, makedirs, fchmod, utime): fchmod.assert_called_once_with(1337, "-rw-rw-r--") utime.assert_called_once_with(key, (7, 7)) self.log.check( - ("pysapcar", "INFO", "d {}".format(dirname)), - ("pysapcar", "INFO", "d {}".format(key)), - ("pysapcar", "INFO", "pysapcar: 1 file(s) processed") + ("pysap.pysapcar", "INFO", "d {}".format(dirname)), + ("pysap.pysapcar", "INFO", "d {}".format(key)), + ("pysap.pysapcar", "INFO", "pysapcar: 1 file(s) processed") ) @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, fchmod=mock.DEFAULT) @@ -324,10 +324,10 @@ def test_extract_file_invalid_file(self, fchmod, utime): fchmod.assert_not_called() utime.assert_not_called() self.log.check( - ("pysapcar", "ERROR", "pysapcar: Invalid SAP CAR file '{}' (Unit test error)" + ("pysap.pysapcar", "ERROR", "pysapcar: Invalid SAP CAR file '{}' (Unit test error)" .format(self.cli.archive_fd.name)), - ("pysapcar", "INFO", "pysapcar: Skipping extraction of file '{}'".format(key)), - ("pysapcar", "INFO", "pysapcar: 0 file(s) processed") + ("pysap.pysapcar", "INFO", "pysapcar: Skipping extraction of file '{}'".format(key)), + ("pysap.pysapcar", "INFO", "pysapcar: 0 file(s) processed") ) @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, fchmod=mock.DEFAULT) @@ -340,10 +340,10 @@ def test_extract_file_invalid_checksum(self, fchmod, utime): fchmod.assert_not_called() utime.assert_not_called() self.log.check( - ("pysapcar", "ERROR", "pysapcar: Invalid checksum found for file '{}'" + ("pysap.pysapcar", "ERROR", "pysapcar: Invalid checksum found for file '{}'" .format(key)), - ("pysapcar", "INFO", "pysapcar: Stopping extraction"), - ("pysapcar", "INFO", "pysapcar: 0 file(s) processed") + ("pysap.pysapcar", "INFO", "pysapcar: Stopping extraction"), + ("pysap.pysapcar", "INFO", "pysapcar: 0 file(s) processed") ) @mock.patch.multiple("pysap.sapcarcli", autospec=True, utime=mock.DEFAULT, fchmod=mock.DEFAULT) @@ -358,11 +358,12 @@ def test_extract_file_extraction_fails(self, fchmod, utime): fchmod.assert_not_called() utime.assert_not_called() self.log.check( - ("pysapcar", "ERROR", "pysapcar: Failed to extract file '{}', (Unit test error)".format(key)), - ("pysapcar", "INFO", "pysapcar: Stopping extraction"), - ("pysapcar", "INFO", "pysapcar: 0 file(s) processed") + ("pysap.pysapcar", "ERROR", "pysapcar: Failed to extract file '{}', ([Errno 13] Unit test error)".format(key)), + ("pysap.pysapcar", "INFO", "pysapcar: Stopping extraction"), + ("pysap.pysapcar", "INFO", "pysapcar: 0 file(s) processed") ) + def test_suite(): loader = unittest.TestLoader() suite = unittest.TestSuite() From 4456850b27681f22e330e15bd107afc597f6b1bc Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Tue, 10 Nov 2020 18:33:34 -0800 Subject: [PATCH 56/74] Python2/3: Fixing exceptions, unit tests and some code --- pysap/SAPLPS.py | 14 +++++++------- pysap/utils/crypto.py | 6 +++--- tests/test_pysapcompress.py | 13 +++++++------ tests/test_sappse.py | 3 ++- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pysap/SAPLPS.py b/pysap/SAPLPS.py index 88e76c0..f729bd1 100644 --- a/pysap/SAPLPS.py +++ b/pysap/SAPLPS.py @@ -36,7 +36,7 @@ log_lps = logging.getLogger("pysap.lps") -cred_key_lps_fallback = "\xe7\x6a\xd2\xce\x4b\xa7\xc7\x9e\xf9\x79\x5f\xa8\x2e\x6e\xaa\x1d\x76\x02\x2e\xcd\xd7\x74\x38\x51" +cred_key_lps_fallback = b"\xe7\x6a\xd2\xce\x4b\xa7\xc7\x9e\xf9\x79\x5f\xa8\x2e\x6e\xaa\x1d\x76\x02\x2e\xcd\xd7\x74\x38\x51" """Fixed key embedded in CommonCryptoLib for encrypted credentials using LPS in fallback mode""" @@ -91,7 +91,7 @@ def decrypt(self): Validation of the checksum and HMAC is not implemented. :return: decrypted object - :rtype: string + :rtype: bytes :raise NotImplementedError: if the LPS method is not implemented :raise SAP_LPS_Decryption_Error: if there's an error decrypting the object @@ -112,7 +112,7 @@ def decrypt(self): raise SAPLPSDecryptionError("Invalid LPS decryption method") # Decrypt the cipher text with the encryption key - iv = "\x00" * 16 + iv = b"\x00" * 16 decryptor = Cipher(algorithms.AES(encryption_key), modes.CBC(iv), backend=default_backend()).decryptor() plain = decryptor.update(self.encrypted_data) + decryptor.finalize() @@ -125,7 +125,7 @@ def decrypt_encryption_key_dpapi(self): the DP API without any additional entropy. :return: Encryption key decrypted - :rtype: string + :rtype: bytes """ log_lps.debug("Obtaining encryption key with DPAPI LPS mode") @@ -138,7 +138,7 @@ def decrypt_encryption_key_fallback(self): encrypt the actual encryption key used in the file with the AES cipher. :return: Encryption key decrypted - :rtype: string + :rtype: bytes """ log_lps.debug("Obtaining encryption key with FALLBACK LPS mode") @@ -150,7 +150,7 @@ def decrypt_encryption_key_fallback(self): hmac.update(self.context) default_key = hmac.finalize()[:16] - iv = "\x00" * 16 + iv = b"\x00" * 16 decryptor = Cipher(algorithms.AES(default_key), modes.CBC(iv), backend=default_backend()).decryptor() encryption_key = decryptor.update(self.encrypted_key) + decryptor.finalize() @@ -160,7 +160,7 @@ def decrypt_encryption_key_tpm(self): """Decrypts the encryption key using the TPM method. :return: Encryption key decrypted - :rtype: string + :rtype: bytes """ log_lps.error("LPS TPM decryption not implemented") raise NotImplementedError("LPS TPM decryption not implemented") diff --git a/pysap/utils/crypto.py b/pysap/utils/crypto.py index b5a53b9..a6548e0 100644 --- a/pysap/utils/crypto.py +++ b/pysap/utils/crypto.py @@ -41,7 +41,7 @@ def dpapi_decrypt_blob(blob, entropy=None): :type entropy: string :return: decrypted blob - :rtype: string + :rtype: bytes :raise Exception: if the platform is not Windows or the decryption failed """ @@ -152,7 +152,7 @@ def to_bytes(value): a = b'\x00' * (c * u) for n in range(1, c + 1): - a2 = digest(d + i) + a2 = digest(d.encode('latin-1') + i) for _ in range(2, self._iterations + 1): a2 = digest(a2) @@ -162,7 +162,7 @@ def to_bytes(value): while len(b) < v: b += a2 - b = to_int(b[:v]) + 1 + b = to_int(str(b)[:v]) + 1 # Step 6.C for n2 in range(0, len(i) // v): diff --git a/tests/test_pysapcompress.py b/tests/test_pysapcompress.py index 02d9de9..2261db4 100755 --- a/tests/test_pysapcompress.py +++ b/tests/test_pysapcompress.py @@ -19,9 +19,10 @@ # Standard imports from __future__ import unicode_literals -import six import sys import unittest +# External imports +from six import assertRaisesRegex # Custom imports from tests.utils import read_data_file @@ -43,15 +44,15 @@ def test_import(self): def test_compress_input(self): """Test compress function input""" from pysapcompress import compress, CompressError - six.assertRaisesRegex(self, CompressError, "invalid input length", compress, b"") - six.assertRaisesRegex(self, CompressError, "unknown algorithm", compress, b"TestString", algorithm=999) + assertRaisesRegex(self, CompressError, "invalid input length", compress, b"") + assertRaisesRegex(self, CompressError, "unknown algorithm", compress, b"TestString", algorithm=999) def test_decompress_input(self): """Test decompress function input""" from pysapcompress import decompress, DecompressError - six.assertRaisesRegex(self, DecompressError, "invalid input length", decompress, b"", 1) - six.assertRaisesRegex(self, DecompressError, "input not compressed", decompress, b"AAAAAAAA", 1) - six.assertRaisesRegex(self, DecompressError, "unknown algorithm", decompress, + assertRaisesRegex(self, DecompressError, "invalid input length", decompress, b"", 1) + assertRaisesRegex(self, DecompressError, "input not compressed", decompress, b"AAAAAAAA", 1) + assertRaisesRegex(self, DecompressError, "unknown algorithm", decompress, b"\x0f\x00\x00\x00\xff\x1f\x9d\x00\x00\x00\x00", 1) def test_lzc(self): diff --git a/tests/test_sappse.py b/tests/test_sappse.py index 87ba4b0..afbc00d 100755 --- a/tests/test_sappse.py +++ b/tests/test_sappse.py @@ -21,6 +21,7 @@ import sys import unittest # External imports +from six import b, assertRaisesRegex # Custom imports from tests.utils import data_filename from pysap.SAPPSE import (SAPPSEFile, PKCS12_ALGORITHM_PBE1_SHA_3DES_CBC) @@ -52,7 +53,7 @@ def test_pse_v2_lps_off_pbes1_3des_sha1_decrypt(self): s = fd.read() pse = SAPPSEFile(s) - self.assertRaisesRegexp(ValueError, "Invalid PIN supplied", pse.decrypt, "Some Invalid PIN") + assertRaisesRegex(self, ValueError, "Invalid PIN supplied", pse.decrypt, "Some Invalid PIN") pse.decrypt(self.decrypt_pin) def test_pse_v4_lps_off_pbes1_3des_sha1(self): From 93da5279ea7164b73bed1d8e3b597a79259d1b36 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Tue, 10 Nov 2020 19:07:31 -0800 Subject: [PATCH 57/74] Python2/3: Making LPS and Credv2 compatible --- pysap/SAPCredv2.py | 34 +++++++++++++++++++--------------- tests/test_sapcredv2.py | 22 +++++++++++----------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/pysap/SAPCredv2.py b/pysap/SAPCredv2.py index af6550e..1363798 100644 --- a/pysap/SAPCredv2.py +++ b/pysap/SAPCredv2.py @@ -22,6 +22,7 @@ import logging from binascii import unhexlify # External imports +import six from scapy.packet import Packet from scapy.asn1packet import ASN1_Packet from scapy.fields import (ByteField, ByteEnumField, ShortField, StrField, StrFixedLenField) @@ -98,7 +99,7 @@ def decrypt_MSCryptProtect(plain, cred): entropy = cred.pse_path return dpapi_decrypt_blob(unhexlify(plain.blob.val), entropy) - PROVIDER_MSCryptProtect = "MSCryptProtect" + PROVIDER_MSCryptProtect = b"MSCryptProtect" """Provider for Windows hosts using DPAPI""" providers = { @@ -168,14 +169,14 @@ def lps_type_str(self): @property def cipher_format_version(self): cipher = self.cipher.val_readable - if len(cipher) >= 36 and ord(cipher[0]) in [0, 1]: - return ord(cipher[0]) + if len(cipher) >= 36 and six.byte2int(cipher) in [0, 1]: + return six.byte2int(cipher) return 0 @property def cipher_algorithm(self): if self.cipher_format_version == 1: - return ord(self.cipher.val_readable[1]) + return six.indexbytes(self.cipher.val_readable, 1) return 0 def decrypt(self, username): @@ -209,9 +210,9 @@ def decrypt_simple(self, username): blob = self.cipher.val_readable # Construct the key using the key format and the username - key = (cred_key_fmt % username)[:24] + key = six.b((cred_key_fmt % username)[:24]) # Set empty IV - iv = "\x00" * 8 + iv = b"\x00" * 8 # Decrypt the cipher text with the derived key and IV decryptor = Cipher(algorithms.TripleDES(key), modes.CBC(iv), backend=default_backend()).decryptor() @@ -252,11 +253,14 @@ def xor(string, start): """XOR a given string using a fixed key and a starting number.""" key = 0x15a4e35 x = start - y = "" + y = b"" for c in string: x *= key x += 1 - y += chr(ord(c) ^ (x & 0xff)) + if six.PY2: + y += chr(ord(c) ^ (x & 0xff)) + elif six.PY3: + y += six.int2byte(c ^ (x & 0xff)) return y def derive_key(key, header, salt, username): @@ -267,14 +271,14 @@ def derive_key(key, header, salt, username): digest.update(key) digest.update(header) digest.update(salt) - digest.update(xor(username, ord(salt[0]))) - digest.update("" * 0x20) + digest.update(xor(username, six.byte2int(salt))) + digest.update(b"" * 0x20) hashed = digest.finalize() - derived_key = xor(hashed, ord(salt[1])) + derived_key = xor(hashed, six.indexbytes(salt, 1)) return derived_key # Derive the key using SAP's algorithm - key = derive_key(cred_key_fmt, blob[0:4], header.salt, username) + key = derive_key(six.b(cred_key_fmt), blob[0:4], header.salt, six.b(username)) # Decrypt the cipher text with the derived key and IV decryptor = Cipher(algorithm(key), modes.CBC(header.iv), backend=default_backend()).decryptor() @@ -313,7 +317,7 @@ def pse_file_path(self): @property def lps_type(self): - return ord(self.cipher.val_readable[1]) + return six.indexbytes(self.cipher.val_readable, 1) @property def lps_type_str(self): @@ -325,7 +329,7 @@ def lps_type_str(self): @property def cipher_format_version(self): - return ord(self.cipher.val_readable[0]) + return six.byte2int(self.cipher.val_readable) @property def cipher_algorithm(self): @@ -350,7 +354,7 @@ def decrypt(self, username=None): plain = cipher.decrypt() # Get the pin from the raw data - plain_size = ord(plain[0]) + plain_size = six.byte2int(plain) pin = plain[plain_size + 1:] # Create a plain credential container diff --git a/tests/test_sapcredv2.py b/tests/test_sapcredv2.py index 255d020..e8ae858 100755 --- a/tests/test_sapcredv2.py +++ b/tests/test_sapcredv2.py @@ -30,11 +30,11 @@ class PySAPCredV2Test(unittest.TestCase): decrypt_username = "username" - decrypt_pin = "1234567890" - cert_name = "CN=PSEOwner" - common_name = "PSEOwner" - pse_path = "/secudir/pse-v2-noreq-DSA-1024-SHA1.pse" - pse_path_win = "C:\\secudir\\pse-v2-noreq-DSA-1024-SHA1.pse" + decrypt_pin = b"1234567890" + cert_name = b"CN=PSEOwner" + common_name = b"PSEOwner" + pse_path = b"/secudir/pse-v2-noreq-DSA-1024-SHA1.pse" + pse_path_win = b"C:\\secudir\\pse-v2-noreq-DSA-1024-SHA1.pse" def test_cred_v2_lps_off_3des(self): """Test parsing of a 3DES encrypted credential with LPS off""" @@ -53,9 +53,9 @@ def test_cred_v2_lps_off_3des(self): self.assertEqual(cred.cipher_algorithm, CIPHER_ALGORITHM_3DES) self.assertEqual(cred.cert_name, self.cert_name) - self.assertEqual(cred.unknown1, "") + self.assertEqual(cred.unknown1, b"") self.assertEqual(cred.pse_path, self.pse_path) - self.assertEqual(cred.unknown2, "") + self.assertEqual(cred.unknown2, b"") def test_cred_v2_lps_off_3des_decrypt(self): """Test decryption of a 3DES encrypted credential with LPS off""" @@ -84,9 +84,9 @@ def test_cred_v2_lps_off_dp_3des(self): self.assertEqual(cred.cipher_algorithm, CIPHER_ALGORITHM_3DES) self.assertEqual(cred.cert_name, self.cert_name) - self.assertEqual(cred.unknown1, "") + self.assertEqual(cred.unknown1, b"") self.assertEqual(cred.pse_path, self.pse_path_win) - self.assertEqual(cred.unknown2, "") + self.assertEqual(cred.unknown2, b"") def test_cred_v2_lps_off_dp_3des_decrypt(self): """Test decryption of a 3DES encrypted credential with LPS off using DP (Windows)""" @@ -115,9 +115,9 @@ def test_cred_v2_lps_off_aes256(self): self.assertEqual(cred.cipher_algorithm, CIPHER_ALGORITHM_AES256) self.assertEqual(cred.cert_name, self.cert_name) - self.assertEqual(cred.unknown1, "") + self.assertEqual(cred.unknown1, b"") self.assertEqual(cred.pse_path, self.pse_path) - self.assertEqual(cred.unknown2, "") + self.assertEqual(cred.unknown2, b"") def test_cred_v2_lps_off_aes256_decrypt(self): """Test decryption of a AES256 encrypted credential with LPS off""" From d4e437b70e885eb2f2ed21cc394806723070335d Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Tue, 10 Nov 2020 19:56:35 -0800 Subject: [PATCH 58/74] Python2/3: Fix binary check on open file --- bin/pysapgenpse | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pysapgenpse b/bin/pysapgenpse index a758b94..b85515f 100755 --- a/bin/pysapgenpse +++ b/bin/pysapgenpse @@ -141,7 +141,7 @@ class PySAPGenPSE(object): """ try: - with open(filename) as f: + with open(filename, 'rb') as f: obj = cls(f.read()) self.logger.info("pysapgenpse: Reading {} file '{}'".format(type, filename)) except IOError as e: From adabf70f5dae6fcb73205732cc005fff900dd90f Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Thu, 25 Mar 2021 19:54:34 -0700 Subject: [PATCH 59/74] Test: Fixed sapcarcli unit test --- tests/test_sapcarcli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_sapcarcli.py b/tests/test_sapcarcli.py index 50e1cce..98ed3a1 100755 --- a/tests/test_sapcarcli.py +++ b/tests/test_sapcarcli.py @@ -355,9 +355,11 @@ def test_extract_file_extraction_fails(self, fchmod, utime): with mock.patch("pysap.sapcarcli.open", mock.mock_open()) as mock_open: mock_open.side_effect = OSError(13, "Unit test error") self.cli.extract(mock.MagicMock(outdir=False), None) + dirname = path.dirname(key) fchmod.assert_not_called() utime.assert_not_called() self.log.check( + ("pysap.pysapcar", "INFO", "d {}".format(dirname)), ("pysap.pysapcar", "ERROR", "pysapcar: Failed to extract file '{}', ([Errno 13] Unit test error)".format(key)), ("pysap.pysapcar", "INFO", "pysapcar: Stopping extraction"), ("pysap.pysapcar", "INFO", "pysapcar: 0 file(s) processed") From 3f8894ed55029e43c2862fbeaf14fa783fa22e59 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Tue, 6 Apr 2021 18:02:47 -0700 Subject: [PATCH 60/74] CI: Start moving to use tox across Python versions --- requirements-tests.txt | 2 ++ requirements.txt | 2 +- setup.py | 5 ----- tox.ini | 15 +++++++++++++++ 4 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 tox.ini diff --git a/requirements-tests.txt b/requirements-tests.txt index c8e2159..201f1bf 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,2 +1,4 @@ testfixtures==6.2.0 mock==2.0.0; python_version < '3' +tox==3.23.0 +pytest \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 27e19e1..6faa008 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -six==1.12 +six==1.15 scapy==2.4.4 cryptography==2.9.2 diff --git a/setup.py b/setup.py index 29d6374..bf4e30a 100755 --- a/setup.py +++ b/setup.py @@ -124,9 +124,6 @@ def run(self): # Script files scripts=['bin/pysapcar', 'bin/pysapgenpse'], - # Tests command - test_suite='tests.test_suite', - # Documentation commands cmdclass={'doc': DocumentationCommand, 'notebooks': PreExecuteNotebooksCommand}, @@ -134,8 +131,6 @@ def run(self): # Requirements install_requires=open('requirements.txt').read().splitlines(), - tests_require=open('requirements-tests.txt').read().splitlines(), - # Optional requirements for docs and some examples extras_require={"docs": open('requirements-docs.txt').read().splitlines(), "examples": open('requirements-examples.txt').read().splitlines()}, diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..be0d67d --- /dev/null +++ b/tox.ini @@ -0,0 +1,15 @@ +[tox] +envlist = py27,py36,py37,py38 + +[testenv] +basepython = + py27: python2.7 + py36: python3.6 + py37: python3.7 + py38: python3.8 + py39: python3.9 +deps=-rrequirements.txt + -rrequirements-tests.txt +passenv = NO_REMOTE +commands_pre = {envpython} -m pip check +commands = pytest \ No newline at end of file From e4c7d76cf3cfaa843c8d602674f88cea888ccbb0 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Mon, 19 Apr 2021 18:45:50 -0700 Subject: [PATCH 61/74] Python2/3: Removed compat errors --- examples/diag_rogue_server.py | 50 +++++++++++++++++------------------ examples/dlmanager_decrypt.py | 2 +- tests/test_pysapcompress.py | 4 +-- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/examples/diag_rogue_server.py b/examples/diag_rogue_server.py index e1b6f16..c8eb00c 100755 --- a/examples/diag_rogue_server.py +++ b/examples/diag_rogue_server.py @@ -84,35 +84,35 @@ def make_login_screen(self): SAPDiagItem(item_value='SAPMSYST ', item_type=16, item_id=6, item_sid=13), SAPDiagItem(item_value='0020 ', item_type=16, item_id=6, item_sid=16), SAPDiagItem(item_value='0020', item_type=16, item_id=6, item_sid=14), - SAPDiagItem(item_value=SAPDiagMenuEntries(entries=[SAPDiagMenuEntry(accelerator='D', text=self.session_title, position_1=1, flag_TERM_VKEY=1L, return_code_1=1, flag_TERM_SEL=1L, length=24 + len(self.session_title)), + SAPDiagItem(item_value=SAPDiagMenuEntries(entries=[SAPDiagMenuEntry(accelerator='D', text=self.session_title, position_1=1, flag_TERM_VKEY=1, return_code_1=1, flag_TERM_SEL=1, length=24 + len(self.session_title)), ]), item_type=18, item_id=11, item_sid=1), - SAPDiagItem(item_value=SAPDiagMenuEntries(entries=[SAPDiagMenuEntry(accelerator='', text='New password', virtual_key=5, return_code_1=5, info='New password', flag_TERM_SEL=1L, length=47), + SAPDiagItem(item_value=SAPDiagMenuEntries(entries=[SAPDiagMenuEntry(accelerator='', text='New password', virtual_key=5, return_code_1=5, info='New password', flag_TERM_SEL=1, length=47), ]), item_type=18, item_id=11, item_sid=3), - SAPDiagItem(item_value=SAPDiagMenuEntries(entries=[SAPDiagMenuEntry(accelerator='', text='New password', virtual_key=5, position_1=1, flag_TERM_SEL=1L, length=35), - SAPDiagMenuEntry(accelerator='', text='Log off', virtual_key=15, position_1=2, return_code_1=1, flag_TERM_SEL=1L, length=30) + SAPDiagItem(item_value=SAPDiagMenuEntries(entries=[SAPDiagMenuEntry(accelerator='', text='New password', virtual_key=5, position_1=1, flag_TERM_SEL=1, length=35), + SAPDiagMenuEntry(accelerator='', text='Log off', virtual_key=15, position_1=2, return_code_1=1, flag_TERM_SEL=1, length=30) ]), item_type=18, item_id=11, item_sid=4), SAPDiagItem(item_value=self.session_title, item_type=16, item_id=12, item_sid=9), - SAPDiagItem(item_value=SAPDiagDyntAtom(items=[SAPDiagDyntAtomItem(field2_text='Client ', field2_maxnrchars=18, dlg_flag_2=2, atom_length=37, etype=132, attr_DIAG_BSD_PROTECTED=1L, field2_mlen=18, field2_dlen=18, attr_DIAG_BSD_PROPFONT=1L, block=1, col=1), - SAPDiagDyntAtomItem(dlg_flag_2=2, atom_length=24, name_text='RSYST-MANDT', etype=114, attr_DIAG_BSD_PROTECTED=1L, attr_DIAG_BSD_PROPFONT=1L, block=1, col=1), - SAPDiagDyntAtomItem(field2_text=self.client, attr_DIAG_BSD_YES3D=1L, field2_maxnrchars=3, atom_length=22, etype=130, field2_mlen=3, field2_dlen=3, block=1, col=20), - SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1L, atom_length=24, name_text='RSYST-MANDT', etype=114, block=1, col=20), - SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1L, atom_length=79, xmlprop_text='Client', etype=120, block=1, col=20), - SAPDiagDyntAtomItem(field2_text='User ', field2_maxnrchars=18, row=2, dlg_flag_2=3, atom_length=37, etype=132, attr_DIAG_BSD_PROTECTED=1L, field2_mlen=18, field2_dlen=18, attr_DIAG_BSD_PROPFONT=1L, block=1, col=1), - SAPDiagDyntAtomItem(row=2, dlg_flag_2=3, atom_length=24, name_text='RSYST-BNAME', etype=114, attr_DIAG_BSD_PROTECTED=1L, attr_DIAG_BSD_PROPFONT=1L, block=1, col=1), - SAPDiagDyntAtomItem(field2_text='? ', attr_DIAG_BSD_YES3D=1L, field2_maxnrchars=12, row=2, dlg_flag_2=1, atom_length=31, etype=130, field2_mlen=12, field2_dlen=12, block=1, col=20), - SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1L, row=2, dlg_flag_2=1, atom_length=24, name_text='RSYST-BNAME', etype=114, block=1, col=20), - SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1L, row=2, dlg_flag_2=1, atom_length=82, xmlprop_text='User name', etype=120, block=1, col=20), - SAPDiagDyntAtomItem(field2_text='@\\QUp to 40 Chars (Case-Sens.)@Password ', field2_maxnrchars=52, row=3, dlg_flag_2=3, dlg_flag_1=4, atom_length=71, etype=132, attr_DIAG_BSD_PROTECTED=1L, field2_mlen=18, field2_dlen=52, attr_DIAG_BSD_PROPFONT=1L, block=1, col=1), - SAPDiagDyntAtomItem(row=3, dlg_flag_2=3, dlg_flag_1=4, atom_length=24, name_text='RSYST-BCODE', etype=114, attr_DIAG_BSD_PROTECTED=1L, attr_DIAG_BSD_PROPFONT=1L, block=1, col=1), - SAPDiagDyntAtomItem(field2_text='? ', attr_DIAG_BSD_YES3D=1L, field2_maxnrchars=40, row=3, dlg_flag_2=1, dlg_flag_1=4, atom_length=59, etype=130, attr_DIAG_BSD_INVISIBLE=1L, field2_mlen=12, field2_dlen=40, block=1, col=20), - SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1L, row=3, dlg_flag_2=1, dlg_flag_1=4, atom_length=24, name_text='RSYST-BCODE', etype=114, attr_DIAG_BSD_INVISIBLE=1L, block=1, col=20), - SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1L, row=3, dlg_flag_2=1, dlg_flag_1=4, atom_length=86, xmlprop_text='User password', etype=120, attr_DIAG_BSD_INVISIBLE=1L, block=1, col=20), - SAPDiagDyntAtomItem(field2_text='Language ', field2_maxnrchars=18, row=5, dlg_flag_2=2, atom_length=37, etype=132, attr_DIAG_BSD_PROTECTED=1L, field2_mlen=18, field2_dlen=18, attr_DIAG_BSD_PROPFONT=1L, block=1, col=1), - SAPDiagDyntAtomItem(row=5, dlg_flag_2=2, atom_length=24, name_text='RSYST-LANGU', etype=114, attr_DIAG_BSD_PROTECTED=1L, attr_DIAG_BSD_PROPFONT=1L, block=1, col=1), - SAPDiagDyntAtomItem(field2_text=' ', attr_DIAG_BSD_YES3D=1L, field2_maxnrchars=2, row=5, atom_length=21, etype=130, field2_mlen=2, field2_dlen=2, block=1, col=20), - SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1L, row=5, atom_length=24, name_text='RSYST-LANGU', etype=114, block=1, col=20), - SAPDiagDyntAtomItem(atom_length=81, attr_DIAG_BSD_YES3D=1L, xmlprop_text='Language', etype=120, col=20, block=1, row=5), - SAPDiagDyntAtomItem(field2_text=self.session_title, field2_maxnrchars=18, row=7, dlg_flag_2=2, atom_length=37, etype=132, attr_DIAG_BSD_PROTECTED=1L, field2_mlen=18, field2_dlen=18, attr_DIAG_BSD_PROPFONT=1L, block=1, col=1), + SAPDiagItem(item_value=SAPDiagDyntAtom(items=[SAPDiagDyntAtomItem(field2_text='Client ', field2_maxnrchars=18, dlg_flag_2=2, atom_length=37, etype=132, attr_DIAG_BSD_PROTECTED=1, field2_mlen=18, field2_dlen=18, attr_DIAG_BSD_PROPFONT=1, block=1, col=1), + SAPDiagDyntAtomItem(dlg_flag_2=2, atom_length=24, name_text='RSYST-MANDT', etype=114, attr_DIAG_BSD_PROTECTED=1, attr_DIAG_BSD_PROPFONT=1, block=1, col=1), + SAPDiagDyntAtomItem(field2_text=self.client, attr_DIAG_BSD_YES3D=1, field2_maxnrchars=3, atom_length=22, etype=130, field2_mlen=3, field2_dlen=3, block=1, col=20), + SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1, atom_length=24, name_text='RSYST-MANDT', etype=114, block=1, col=20), + SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1, atom_length=79, xmlprop_text='Client', etype=120, block=1, col=20), + SAPDiagDyntAtomItem(field2_text='User ', field2_maxnrchars=18, row=2, dlg_flag_2=3, atom_length=37, etype=132, attr_DIAG_BSD_PROTECTED=1, field2_mlen=18, field2_dlen=18, attr_DIAG_BSD_PROPFONT=1, block=1, col=1), + SAPDiagDyntAtomItem(row=2, dlg_flag_2=3, atom_length=24, name_text='RSYST-BNAME', etype=114, attr_DIAG_BSD_PROTECTED=1, attr_DIAG_BSD_PROPFONT=1, block=1, col=1), + SAPDiagDyntAtomItem(field2_text='? ', attr_DIAG_BSD_YES3D=1, field2_maxnrchars=12, row=2, dlg_flag_2=1, atom_length=31, etype=130, field2_mlen=12, field2_dlen=12, block=1, col=20), + SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1, row=2, dlg_flag_2=1, atom_length=24, name_text='RSYST-BNAME', etype=114, block=1, col=20), + SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1, row=2, dlg_flag_2=1, atom_length=82, xmlprop_text='User name', etype=120, block=1, col=20), + SAPDiagDyntAtomItem(field2_text='@\\QUp to 40 Chars (Case-Sens.)@Password ', field2_maxnrchars=52, row=3, dlg_flag_2=3, dlg_flag_1=4, atom_length=71, etype=132, attr_DIAG_BSD_PROTECTED=1, field2_mlen=18, field2_dlen=52, attr_DIAG_BSD_PROPFONT=1, block=1, col=1), + SAPDiagDyntAtomItem(row=3, dlg_flag_2=3, dlg_flag_1=4, atom_length=24, name_text='RSYST-BCODE', etype=114, attr_DIAG_BSD_PROTECTED=1, attr_DIAG_BSD_PROPFONT=1, block=1, col=1), + SAPDiagDyntAtomItem(field2_text='? ', attr_DIAG_BSD_YES3D=1, field2_maxnrchars=40, row=3, dlg_flag_2=1, dlg_flag_1=4, atom_length=59, etype=130, attr_DIAG_BSD_INVISIBLE=1, field2_mlen=12, field2_dlen=40, block=1, col=20), + SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1, row=3, dlg_flag_2=1, dlg_flag_1=4, atom_length=24, name_text='RSYST-BCODE', etype=114, attr_DIAG_BSD_INVISIBLE=1, block=1, col=20), + SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1, row=3, dlg_flag_2=1, dlg_flag_1=4, atom_length=86, xmlprop_text='User password', etype=120, attr_DIAG_BSD_INVISIBLE=1, block=1, col=20), + SAPDiagDyntAtomItem(field2_text='Language ', field2_maxnrchars=18, row=5, dlg_flag_2=2, atom_length=37, etype=132, attr_DIAG_BSD_PROTECTED=1, field2_mlen=18, field2_dlen=18, attr_DIAG_BSD_PROPFONT=1, block=1, col=1), + SAPDiagDyntAtomItem(row=5, dlg_flag_2=2, atom_length=24, name_text='RSYST-LANGU', etype=114, attr_DIAG_BSD_PROTECTED=1, attr_DIAG_BSD_PROPFONT=1, block=1, col=1), + SAPDiagDyntAtomItem(field2_text=' ', attr_DIAG_BSD_YES3D=1, field2_maxnrchars=2, row=5, atom_length=21, etype=130, field2_mlen=2, field2_dlen=2, block=1, col=20), + SAPDiagDyntAtomItem(attr_DIAG_BSD_YES3D=1, row=5, atom_length=24, name_text='RSYST-LANGU', etype=114, block=1, col=20), + SAPDiagDyntAtomItem(atom_length=81, attr_DIAG_BSD_YES3D=1, xmlprop_text='Language', etype=120, col=20, block=1, row=5), + SAPDiagDyntAtomItem(field2_text=self.session_title, field2_maxnrchars=18, row=7, dlg_flag_2=2, atom_length=37, etype=132, attr_DIAG_BSD_PROTECTED=1, field2_mlen=18, field2_dlen=18, attr_DIAG_BSD_PROPFONT=1, block=1, col=1), ]), item_type=18, item_id=9, item_sid=2), ] diff --git a/examples/dlmanager_decrypt.py b/examples/dlmanager_decrypt.py index 8a7d14d..77316a3 100755 --- a/examples/dlmanager_decrypt.py +++ b/examples/dlmanager_decrypt.py @@ -178,7 +178,7 @@ def parse_config_file(filename, decrypt=False, serial_number=None): print("[*] Read %d config values from config file" % len(data)) for item in data: value = "" - if isinstance(data[item], basestring): + if isinstance(data[item], str): value = data[item] elif "value" in data[item]: value = data[item]["value"] diff --git a/tests/test_pysapcompress.py b/tests/test_pysapcompress.py index cc920c4..4edb3e4 100755 --- a/tests/test_pysapcompress.py +++ b/tests/test_pysapcompress.py @@ -22,7 +22,7 @@ import sys import unittest # External imports -from six import assertRaisesRegex +from six import assertRaisesRegex, text_type # Custom imports from tests.utils import read_data_file @@ -39,7 +39,7 @@ def test_import(self): try: import pysapcompress # @UnusedImport # noqa: F401 except ImportError as e: - self.fail(six.text_type(e)) + self.fail(text_type(e)) def test_compress_input(self): """Test compress function input""" From 97be156cf34f030fc2a540455825ae0803f2a307 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Mon, 19 Apr 2021 19:45:00 -0700 Subject: [PATCH 62/74] Python2/3: Working on SAPHDB layer compat --- examples/hdb_discovery.py | 3 ++- pysap/SAPHDB.py | 31 ++++++++++++++++--------------- tests/test_saphdb.py | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/examples/hdb_discovery.py b/examples/hdb_discovery.py index 2caeafb..d0ef7a1 100755 --- a/examples/hdb_discovery.py +++ b/examples/hdb_discovery.py @@ -25,6 +25,7 @@ from argparse import ArgumentParser from socket import error as SocketError # External imports +from six import iteritems from scapy.config import conf # Custom imports import pysap @@ -145,7 +146,7 @@ def main(): # List the values in the returned DBConnectInfo part hdb_dbconnectinfo_response_part = hdb_dbconnectinfo_response.segments[0].parts[0] - for key, name in SAPHDBPartDBConnectInfo.option_keys.items(): + for key, name in iteritems(SAPHDBPartDBConnectInfo.option_keys): value = hdb_get_part_kind_option(hdb_dbconnectinfo_response_part, key) if value is not None: results[tenant][name] = value diff --git a/pysap/SAPHDB.py b/pysap/SAPHDB.py index d1eaf6c..7b4bc70 100644 --- a/pysap/SAPHDB.py +++ b/pysap/SAPHDB.py @@ -22,6 +22,7 @@ import socket import struct # External imports +from six import b from cryptography.hazmat.backends import default_backend from scapy.layers.inet import TCP from scapy.packet import Packet, bind_layers, Raw @@ -802,7 +803,7 @@ class SAPHDBInitializationRequest(Packet): """ name = "SAP HANA SQL Command Network Protocol Initialization Request" fields_desc = [ - StrFixedLenField("initialization", "\xff\xff\xff\xff\x04\x20\x00\x04\x01\x00\x00\x01\x01\x01", 14), + StrFixedLenField("initialization", b("\xff\xff\xff\xff\x04\x20\x00\x04\x01\x00\x00\x01\x01\x01"), 14), ] @@ -851,7 +852,7 @@ def craft_authentication_request(self, value=None, connection=None): provided, it will include the Client Context part from it (e.g. application name). :param value: value to include in the authentication request - :type value: string + :type value: bytes :param connection: `HDB` connection to get client context values if required :type connection: :class:`SAPHDBConnection` @@ -861,7 +862,7 @@ def craft_authentication_request(self, value=None, connection=None): """ auth_fields = SAPHDBPartAuthentication(auth_fields=[SAPHDBPartAuthenticationField(value=self.username), SAPHDBPartAuthenticationField(value=self.METHOD), - SAPHDBPartAuthenticationField(value=value or "")]) + SAPHDBPartAuthenticationField(value=value or b(""))]) auth_part = SAPHDBPart(partkind=33, argumentcount=1, buffer=auth_fields) auth_segm = SAPHDBSegment(messagetype=65, parts=[auth_part]) @@ -877,14 +878,14 @@ def craft_authentication_response_part(self, auth_response_part, value=None): :type auth_response_part: :class:`SAPHDBPartAuthentication` :param value: value to include in the authentication response - :type value: string + :type value: bytes :return: the authentication response part :rtype: :class:`SAPHDBPart` """ auth_fields = SAPHDBPartAuthentication(auth_fields=[SAPHDBPartAuthenticationField(value=self.username), SAPHDBPartAuthenticationField(value=self.METHOD), - SAPHDBPartAuthenticationField(value=value or "")]) + SAPHDBPartAuthenticationField(value=value or b(""))]) return SAPHDBPart(partkind=33, argumentcount=1, buffer=auth_fields) def authenticate(self, connection): @@ -955,7 +956,7 @@ def obtain_client_proof(self, scram, client_key, auth_response_part): # Calculate the client proof from the password, salt and the server and client key # TODO: It might be good to see if this can be moved into a new Packet # TODO: We're only considering one server key - client_proof = b"\x00\x01" + struct.pack('b', scram.CLIENT_PROOF_SIZE) + client_proof = b("\x00\x01") + struct.pack('b', scram.CLIENT_PROOF_SIZE) client_proof += scram.scramble_salt(self.password, salt, server_key, client_key) return client_proof @@ -1001,7 +1002,7 @@ def obtain_client_proof(self, scram, client_key, auth_response_part): # Calculate the client proof from the password, salt, rounds and the server and client key # TODO: It might be good to see if this can be moved into a new Packet # TODO: We're only considering one server key - client_proof = b"\x00\x01" + struct.pack('b', scram.CLIENT_PROOF_SIZE) + client_proof = b("\x00\x01") + struct.pack('b', scram.CLIENT_PROOF_SIZE) client_proof += scram.scramble_salt(self.password, salt, server_key, client_key, rounds) return client_proof @@ -1086,7 +1087,7 @@ def craft_authentication_request(self, value=None, connection=None): doesn't use the standard ASN.1 encoding and instead leverage the same Authentication Field format. """ - auth_fields = SAPHDBPartAuthentication(auth_fields=[SAPHDBPartAuthenticationField(value=""), + auth_fields = SAPHDBPartAuthentication(auth_fields=[SAPHDBPartAuthenticationField(value=b("")), SAPHDBPartAuthenticationField(value=self.METHOD), SAPHDBPartAuthenticationField(value=value)]) auth_part = SAPHDBPart(partkind=33, argumentcount=1, buffer=auth_fields) @@ -1109,7 +1110,7 @@ def craft_authentication_response_part(self, auth_response_part, value=None): # gss_token_response = SAPHDBPartAuthentication(auth_response_part.auth_fields[1].value) last_gss_token = SAPHDBPartAuthentication(auth_fields=[SAPHDBPartAuthenticationField(value=self.krb5oid), - SAPHDBPartAuthenticationField(value="\x05")]) + SAPHDBPartAuthenticationField(value=b("\x05"))]) return super(SAPHDBAuthGSSMethod, self).craft_authentication_response_part(auth_response_part, last_gss_token) def authenticate(self, connection): @@ -1131,7 +1132,7 @@ def authenticate(self, connection): # * typeoid: type of the client GSS name # * gssname: the client GSS name (e.g. UPN) first_gss_token = SAPHDBPartAuthentication(auth_fields=[SAPHDBPartAuthenticationField(value=self.krb5oid), - SAPHDBPartAuthenticationField(value="\x01"), + SAPHDBPartAuthenticationField(value=b("\x01")), SAPHDBPartAuthenticationField(value=self.typeoid), SAPHDBPartAuthenticationField(value=self.username)]) first_auth_request = self.craft_authentication_request(first_gss_token, connection=connection) @@ -1171,7 +1172,7 @@ def authenticate(self, connection): # * commtype: communication type ("\x03") # * krb5ticket: the GSSAPI KRB5 AP-REQ structure to use second_gss_value = SAPHDBPartAuthentication(auth_fields=[SAPHDBPartAuthenticationField(value=self.krb5oid), - SAPHDBPartAuthenticationField(value="\x03"), + SAPHDBPartAuthenticationField(value=b("\x03")), SAPHDBPartAuthenticationField(value=krb5ticket)]) second_auth_request = self.craft_authentication_request(second_gss_value, connection=connection) second_auth_response = connection.sr(second_auth_request) @@ -1200,7 +1201,7 @@ def process_connect_response(self, connect_reponse, connection=None): # * commtype: communication type ("\x07") # * session cookie: the SessionCookie established for the connection gss_token = SAPHDBPartAuthentication(self.session_cookie) - if gss_token.auth_fields[1].value == "\x07": + if gss_token.auth_fields[1].value == b("\x07"): self.session_cookie = gss_token.auth_fields[2].value else: self.session_cookie = None @@ -1335,8 +1336,8 @@ def recv(self): header_raw = self._stream_socket.ins.recv(32) header = SAPHDB(header_raw) # Then get the payload - payload = "" - if header.varpartlength > 0: + payload = b("") + if header.varpartlength and header.varpartlength > 0: payload = self._stream_socket.ins.recv(header.varpartlength) # And finally construct the whole packet with header plus payload return SAPHDB(header_raw + payload) @@ -1355,7 +1356,7 @@ def initialize(self): self.send(init_request) # Receive initialization response packet - init_reply = SAPHDBInitializationReply(self._stream_socket.recv(8)) # We use the raw socket recv here + init_reply = SAPHDBInitializationReply(bytes(self._stream_socket.recv(8))) # We use the raw socket recv here self.product_version = init_reply.product_major self.protocol_version = init_reply.protocol_major diff --git a/tests/test_saphdb.py b/tests/test_saphdb.py index 5091502..5edcfb0 100755 --- a/tests/test_saphdb.py +++ b/tests/test_saphdb.py @@ -32,7 +32,7 @@ class SAPHDBServerTestHandler(BaseRequestHandler): def handle_data(self): self.request.recv(14) - self.request.send("\x00" * 8) + self.request.send(b"\x00" * 8) class PySAPHDBConnectionTest(unittest.TestCase): From e594eb196fa9608f7f69046eeccfcd5ab2ba0d77 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Wed, 28 Apr 2021 14:20:24 -0700 Subject: [PATCH 63/74] CI: Testing requires testing requirements --- .github/workflows/build_and_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 5dde12a..d0bf41a 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -67,6 +67,7 @@ jobs: run: | python -m pip install --upgrade pip wheel pip install -r requirements.txt + pip install -r requirements-tests.txt - name: Run unit tests run: | @@ -101,6 +102,7 @@ jobs: run: | python -m pip install --upgrade pip wheel pip install -r requirements-docs.txt + - name: Install the library run: | python setup.py install From c5277129d51adaca0fb24e98893e5dc24f360ade Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Thu, 29 Apr 2021 04:52:43 -0700 Subject: [PATCH 64/74] Python2/3: Arranged imports --- tests/test_sapcarcli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_sapcarcli.py b/tests/test_sapcarcli.py index 98ed3a1..8e9bf9b 100755 --- a/tests/test_sapcarcli.py +++ b/tests/test_sapcarcli.py @@ -20,21 +20,21 @@ # Standard imports from __future__ import unicode_literals, absolute_import import unittest - +from os import path try: from unittest import mock except ImportError: import mock - -from os import path - +# External imports from testfixtures import LogCapture +# Custom imports from tests.utils import data_filename from pysap.sapcarcli import PySAPCAR from pysap.SAPCAR import SAPCARArchive, SAPCARArchiveFile, SAPCARInvalidFileException, SAPCARInvalidChecksumException class PySAPCARCLITest(unittest.TestCase): + def setUp(self): self.mock_file = mock.Mock( spec=SAPCARArchiveFile, From b08453962e2ba97cd762c73a589226528334d1ab Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Thu, 29 Apr 2021 09:05:34 -0700 Subject: [PATCH 65/74] Arranging ChangeLog --- ChangeLog.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index 5ba72a1..db7fae5 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -5,11 +5,12 @@ Changelog v0.2.1 - 2021-XX-XX ------------------- +- Python 2/3 compatible codebase + v0.1.19 - 2021-04-29 -------------------- -- Python 2/3 compatible codebase - Using Scapy version 2.4.4. - `pysap/SAPSSFS.py`: New module for SAP Secure Store in File System file format. - `bin/pysaphdbuserstore`: New script for interacting with `hdbuserstore` SSFS files. From 69645b4a685b2d5a9cefbdae63ae54c52e5e0c10 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Fri, 30 Apr 2021 13:37:10 -0700 Subject: [PATCH 66/74] CI: Removing Python3.8 in Windows for now as it will fail --- .github/workflows/build_and_test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d0bf41a..5de2731 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -43,9 +43,6 @@ jobs: - os: windows-latest python-version: 2.7 experimental: true - - os: windows-latest - python-version: 3.8 - experimental: true steps: - name: Checkout pysap uses: actions/checkout@v2 From 95a265356fa91bc437ad3fcb599c839f01f53ef8 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Sun, 16 May 2021 15:24:38 -0700 Subject: [PATCH 67/74] Tests: Solve test tearDown --- tests/test_sapcar.py | 2 ++ tests/test_saprouter.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_sapcar.py b/tests/test_sapcar.py index e28291d..b99633b 100755 --- a/tests/test_sapcar.py +++ b/tests/test_sapcar.py @@ -48,6 +48,8 @@ def tearDown(self): for filename in [self.test_filename, self.test_archive_file]: if path.exists(filename): unlink(filename) + if path.exists(path.join("test", "blah")): + unlink(path.join("test", "blah")) if path.exists("test"): rmdir("test") diff --git a/tests/test_saprouter.py b/tests/test_saprouter.py index d304739..574e1c5 100755 --- a/tests/test_saprouter.py +++ b/tests/test_saprouter.py @@ -106,7 +106,6 @@ def handle_data(self): self.packet.decode_payload_as(SAPRouter) route_request = self.packet[SAPRouter] - route_request.show() if router_is_route(route_request): if route_request.route_string[1].hostname == "10.0.0.1" and \ From ffbc780698c023cadc47d2d65ad599f2ce4d3263 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Sun, 16 May 2021 15:25:50 -0700 Subject: [PATCH 68/74] Tests: Arranging test files --- tests/test_crypto.py | 0 tests/{sapssfs_test.py => test_sapssfs.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/test_crypto.py rename tests/{sapssfs_test.py => test_sapssfs.py} (100%) diff --git a/tests/test_crypto.py b/tests/test_crypto.py old mode 100644 new mode 100755 diff --git a/tests/sapssfs_test.py b/tests/test_sapssfs.py similarity index 100% rename from tests/sapssfs_test.py rename to tests/test_sapssfs.py From a19afe4b645096078e377c170997f984737dcb56 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Sun, 16 May 2021 15:34:10 -0700 Subject: [PATCH 69/74] Setup: Bumping testfixtures to avoid warning --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 201f1bf..44bacdf 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,4 +1,4 @@ -testfixtures==6.2.0 +testfixtures==6.17.1 mock==2.0.0; python_version < '3' tox==3.23.0 pytest \ No newline at end of file From bf70616f558b1642f341b6ba966292e31ad2dafd Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Sun, 16 May 2021 15:37:44 -0700 Subject: [PATCH 70/74] Setup: Bump mock version --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 44bacdf..d2b3170 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,4 +1,4 @@ testfixtures==6.17.1 -mock==2.0.0; python_version < '3' +mock==3.0.5; python_version < '3' tox==3.23.0 pytest \ No newline at end of file From 7fbc4f3c8a2a45955527fe91cbb8b86e71471a07 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Sun, 16 May 2021 15:38:08 -0700 Subject: [PATCH 71/74] Setup: Bump tox version --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index d2b3170..6860787 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,4 +1,4 @@ testfixtures==6.17.1 mock==3.0.5; python_version < '3' -tox==3.23.0 +tox==3.23.1 pytest \ No newline at end of file From 56a6a1d6c83e1f420b0cb512eafa6dc67a062397 Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Mon, 16 Aug 2021 15:29:08 -0700 Subject: [PATCH 72/74] SSFS: Some Python3 compat issues solved --- pysap/SAPSSFS.py | 10 +++++----- pysap/utils/crypto/__init__.py | 12 +++++++++--- tests/test_sapssfs.py | 22 ++++++++++++---------- tox.ini | 2 +- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/pysap/SAPSSFS.py b/pysap/SAPSSFS.py index a916245..edc28ae 100644 --- a/pysap/SAPSSFS.py +++ b/pysap/SAPSSFS.py @@ -21,6 +21,7 @@ # Standard imports import logging # External imports +from six import b from scapy.packet import Packet from scapy.fields import (ByteField, YesNoByteField, LenField, StrFixedLenField, StrField, PacketListField) from cryptography.exceptions import InvalidSignature @@ -36,7 +37,7 @@ log_ssfs = logging.getLogger("pysap.ssfs") -ssfs_hmac_key_unobscured = "\xe3\xa0\x61\x11\x85\x41\x68\x99\xf3\x0e\xda\x87\x7a\x80\xcc\x69" +ssfs_hmac_key_unobscured = b("\xe3\xa0\x61\x11\x85\x41\x68\x99\xf3\x0e\xda\x87\x7a\x80\xcc\x69") """Fixed key embedded in rsecssfx binaries for validating integrity of records""" @@ -159,10 +160,9 @@ def decrypt_data(self, key): @property def valid(self): """Returns whether the HMAC-SHA1 value is valid for the given payload""" - # Calculate the HMAC-SHA1 h = HMAC(ssfs_hmac_key_unobscured, SHA1(), backend=default_backend()) - h.update(str(self)[24:156]) # Entire Data header without the HMAC field + h.update(bytes(self)[24:156]) # Entire Data header without the HMAC field h.update(self.data) # Validate the signature @@ -198,7 +198,7 @@ def has_record(self, key_name): :rtype: bool """ for record in self.records: - if record.key_name.rstrip(" ") == key_name: + if record.key_name.rstrip(b(" ")) == key_name: return True return False @@ -212,7 +212,7 @@ def get_records(self, key_name): :rtype: SAPSSFSDataRecord """ for record in self.records: - if record.key_name.rstrip(" ") == key_name: + if record.key_name.rstrip(b(" ")) == key_name: yield record def get_record(self, key_name): diff --git a/pysap/utils/crypto/__init__.py b/pysap/utils/crypto/__init__.py index b857bac..d94651c 100644 --- a/pysap/utils/crypto/__init__.py +++ b/pysap/utils/crypto/__init__.py @@ -21,7 +21,7 @@ import os import math # Custom imports -from rsec import RSECCipher +from .rsec import RSECCipher # External imports from six import b from six.moves import xrange @@ -358,8 +358,14 @@ def rsec_decrypt(blob, key): if len(key) != 24: raise Exception("Wrong key length") - blob = [ord(i) for i in blob] - key = [ord(i) for i in key] + try: + blob = [ord(i) for i in blob] + except TypeError: + blob = [i for i in blob] + try: + key = [ord(i) for i in key] + except TypeError: + key = [i for i in key] key1 = key[0:8] key2 = key[8:16] key3 = key[16:24] diff --git a/tests/test_sapssfs.py b/tests/test_sapssfs.py index 25fb9ec..fc93322 100755 --- a/tests/test_sapssfs.py +++ b/tests/test_sapssfs.py @@ -20,6 +20,7 @@ # Standard imports import unittest # External imports +from six import b # Custom imports from tests.utils import data_filename from pysap.SAPSSFS import (SAPSSFSKey, SAPSSFSData, SAPSSFSLock) @@ -27,8 +28,8 @@ class PySAPSSFSKeyTest(unittest.TestCase): - USERNAME = "SomeUser " - HOST = "ubuntu " + USERNAME = b("SomeUser ") + HOST = b("ubuntu ") def test_ssfs_key_parsing(self): """Test parsing of a SSFS Key file""" @@ -38,7 +39,7 @@ def test_ssfs_key_parsing(self): key = SAPSSFSKey(s) - self.assertEqual(key.preamble, "RSecSSFsKey") + self.assertEqual(key.preamble, b("RSecSSFsKey")) self.assertEqual(key.type, 1) self.assertEqual(key.user, self.USERNAME) self.assertEqual(key.host, self.HOST) @@ -46,12 +47,12 @@ def test_ssfs_key_parsing(self): class PySAPSSFSDataTest(unittest.TestCase): - USERNAME = "SomeUser " - HOST = "ubuntu " + USERNAME = b("SomeUser ") + HOST = b("ubuntu ") - PLAIN_VALUES = {"HDB/KEYNAME/DB_CON_ENV": "Env", - "HDB/KEYNAME/DB_DATABASE_NAME": "Database", - "HDB/KEYNAME/DB_USER": "SomeUser", + PLAIN_VALUES = {b("HDB/KEYNAME/DB_CON_ENV"): b("Env"), + b("HDB/KEYNAME/DB_DATABASE_NAME"): b("Database"), + b("HDB/KEYNAME/DB_USER"): b("SomeUser"), } def test_ssfs_data_parsing(self): @@ -64,7 +65,7 @@ def test_ssfs_data_parsing(self): self.assertEqual(len(data.records), 4) for record in data.records: - self.assertEqual(record.preamble, "RSecSSFsData") + self.assertEqual(record.preamble, b("RSecSSFsData")) self.assertEqual(record.length, len(record)) self.assertEqual(record.type, 1) self.assertEqual(record.user, self.USERNAME) @@ -103,6 +104,7 @@ def test_ssfs_data_record_hmac(self): # Now tamper with the header original_user = record.user record.user = "NewUser" + record.show() self.assertFalse(record.valid) record.user = original_user self.assertTrue(record.valid) @@ -124,7 +126,7 @@ def test_ssfs_data_record_hmac(self): class PySAPSSFSDataDecryptTest(unittest.TestCase): - ENCRYPTED_VALUES = {"HDB/KEYNAME/DB_PASSWORD": "SomePassword"} + ENCRYPTED_VALUES = {b("HDB/KEYNAME/DB_PASSWORD"): b("SomePassword")} def test_ssfs_data_record_decrypt(self): """Test decrypting a record with a given key in a SSFS Data file.""" diff --git a/tox.ini b/tox.ini index be0d67d..d88b43c 100644 --- a/tox.ini +++ b/tox.ini @@ -12,4 +12,4 @@ deps=-rrequirements.txt -rrequirements-tests.txt passenv = NO_REMOTE commands_pre = {envpython} -m pip check -commands = pytest \ No newline at end of file +commands = {envpython} -m pytest {posargs} \ No newline at end of file From bff705b5babe7142615781b78d3ba1dcea93f79d Mon Sep 17 00:00:00 2001 From: Martin Gallo Date: Mon, 16 Aug 2021 15:56:23 -0700 Subject: [PATCH 73/74] Python2/3: Padding bytes and some docstrings --- pysap/SAPSSFS.py | 26 +++++++++++++------------- tests/test_sapssfs.py | 9 ++++----- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/pysap/SAPSSFS.py b/pysap/SAPSSFS.py index edc28ae..9786858 100644 --- a/pysap/SAPSSFS.py +++ b/pysap/SAPSSFS.py @@ -61,8 +61,8 @@ class SAPSSFSLock(Packet): ByteField("file_type", 0), ByteField("type", 0), TimestampField("timestamp", None), - StrFixedLenPaddedField("user", None, 24, padd=" "), - StrFixedLenPaddedField("host", None, 24, padd=" "), + StrFixedLenPaddedField("user", None, 24, padd=b(" ")), + StrFixedLenPaddedField("host", None, 24, padd=b(" ")), ] @@ -77,8 +77,8 @@ class SAPSSFSKey(Packet): ByteField("type", 1), StrFixedLenField("key", None, 24), TimestampField("timestamp", None), - StrFixedLenPaddedField("user", None, 24, padd=" "), - StrFixedLenPaddedField("host", None, 24, padd=" "), + StrFixedLenPaddedField("user", None, 24, padd=b(" ")), + StrFixedLenPaddedField("host", None, 24, padd=b(" ")), ] @@ -102,7 +102,7 @@ class SAPSSFSDecryptedPayload(PacketNoPadded): @property def valid(self): """Returns whether the SHA1 value is valid for the given payload""" - blob = str(self) + blob = bytes(self) digest = Hash(SHA1(), backend=default_backend()) digest.update(blob[:8]) @@ -130,10 +130,10 @@ class SAPSSFSDataRecord(PacketNoPadded): ByteField("type", 1), # Record type "1" supported StrFixedLenField("filler1", None, 7), # Data Header - StrFixedLenPaddedField("key_name", None, 64, padd=" "), + StrFixedLenPaddedField("key_name", None, 64, padd=b(" ")), TimestampField("timestamp", None), - StrFixedLenPaddedField("user", None, 24, padd=" "), - StrFixedLenPaddedField("host", None, 24, padd=" "), + StrFixedLenPaddedField("user", None, 24, padd=b(" ")), + StrFixedLenPaddedField("host", None, 24, padd=b(" ")), YesNoByteField("is_deleted", 0), YesNoByteField("is_stored_as_plaintext", 0), YesNoByteField("is_binary_data", 0), @@ -154,7 +154,7 @@ def decrypt_data(self, key): log_ssfs.debug("Decrypting record {}".format(self.key_name)) decrypted_data = rsec_decrypt(self.data, key.key) decrypted_payload = SAPSSFSDecryptedPayload(decrypted_data) - log_ssfs.warn("Decrypted payload integrity is {}".format(decrypted_payload.valid)) + log_ssfs.warning("Decrypted payload integrity is {}".format(decrypted_payload.valid)) return decrypted_payload.data @property @@ -192,7 +192,7 @@ def has_record(self, key_name): """Returns if the data file contains a record with a given key name. :param key_name: the name of the key to look for - :type key_name: string + :type key_name: bytes :return: if the data file contains the record with key_name :rtype: bool @@ -206,7 +206,7 @@ def get_records(self, key_name): """Generator to retrieve records with the given key name. :param key_name: the name of the key to look for - :type key_name: string + :type key_name: bytes :return: the record with key_name :rtype: SAPSSFSDataRecord @@ -219,7 +219,7 @@ def get_record(self, key_name): """Returns the first record with the given key name. :param key_name: the name of the key to look for - :type key_name: string + :type key_name: bytes :return: the record with key_name :rtype: SAPSSFSDataRecord @@ -230,7 +230,7 @@ def get_value(self, key_name, key=None): """Returns the value with the given key name. :param key_name: the name of the key to look for - :type key_name: string + :type key_name: bytes :param key: the encryption key :type key: SAPSSFSKey diff --git a/tests/test_sapssfs.py b/tests/test_sapssfs.py index fc93322..78fa781 100755 --- a/tests/test_sapssfs.py +++ b/tests/test_sapssfs.py @@ -23,7 +23,7 @@ from six import b # Custom imports from tests.utils import data_filename -from pysap.SAPSSFS import (SAPSSFSKey, SAPSSFSData, SAPSSFSLock) +from pysap.SAPSSFS import (SAPSSFSKey, SAPSSFSData) class PySAPSSFSKeyTest(unittest.TestCase): @@ -103,22 +103,21 @@ def test_ssfs_data_record_hmac(self): # Now tamper with the header original_user = record.user - record.user = "NewUser" - record.show() + record.user = b("NewUser") self.assertFalse(record.valid) record.user = original_user self.assertTrue(record.valid) # Now tamper with the data orginal_data = record.data - record.data = orginal_data + "AddedDataBytes" + record.data = orginal_data + b("AddedDataBytes") self.assertFalse(record.valid) record.data = orginal_data self.assertTrue(record.valid) # Now tamper with the HMAC orginal_hmac = record.hmac - record.hmac = orginal_hmac[:-1] + "A" + record.hmac = orginal_hmac[:-1] + b("A") self.assertFalse(record.valid) record.hmac = orginal_hmac self.assertTrue(record.valid) From 99aa27b05f3eae64b5c08ff665d9c2aad81ef3bc Mon Sep 17 00:00:00 2001 From: topias Date: Thu, 30 Jun 2022 09:15:48 +0300 Subject: [PATCH 74/74] Make pysapcompress compatible with Python 3.10 --- pysapcompress/pysapcompress.cpp | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/pysapcompress/pysapcompress.cpp b/pysapcompress/pysapcompress.cpp index 24663e8..87e3e23 100644 --- a/pysapcompress/pysapcompress.cpp +++ b/pysapcompress/pysapcompress.cpp @@ -21,12 +21,16 @@ # ============== */ +/* Python 3.10 requires PY_SSIZE_T_CLEAN macro when using PyArg_ParseTuple() with "s#" format */ +#define PY_SSIZE_T_CLEAN + #include #include #include #include #include +#include #include "hpa101saptype.h" #include "hpa104CsObject.h" @@ -382,6 +386,7 @@ pysapcompress_compress(PyObject *self, PyObject *args, PyObject *keywds) const unsigned char *in = NULL; unsigned char *out = NULL; int status = 0, in_length = 0, out_length = 0, algorithm = ALG_LZC; + Py_ssize_t in_length_arg = 0; /* Define the keyword list */ static char kwin[] = "in"; @@ -389,9 +394,16 @@ pysapcompress_compress(PyObject *self, PyObject *args, PyObject *keywds) static char* kwlist[] = {kwin, kwalgorithm, NULL}; /* Parse the parameters. We are also interested in the length of the input buffer. */ - if (!PyArg_ParseTupleAndKeywords(args, keywds, BYTES_FORMAT"|i", kwlist, &in, &in_length, &algorithm)) + if (!PyArg_ParseTupleAndKeywords(args, keywds, BYTES_FORMAT"|i", kwlist, &in, &in_length_arg, &algorithm)) return (NULL); + /* Check the size of length args and convert from Py_ssize_t to int */ + if (in_length_arg > INT_MAX) { + return (PyErr_Format(compression_exception, "Compression error (Input length is larger than INT_MAX)")); + } + + in_length = Py_SAFE_DOWNCAST(in_length_arg, Py_ssize_t, int); + /* Call the compression function */ status = compress_packet(in, in_length, &out, &out_length, algorithm); @@ -425,11 +437,21 @@ pysapcompress_decompress(PyObject *self, PyObject *args) const unsigned char *in = NULL; unsigned char *out = NULL; int status = 0, in_length = 0, out_length = 0; + Py_ssize_t in_length_arg = 0, out_length_arg = 0; /* Parse the parameters. We are also interested in the length of the input buffer. */ - if (!PyArg_ParseTuple(args, BYTES_FORMAT"i", &in, &in_length, &out_length)) + if (!PyArg_ParseTuple(args, BYTES_FORMAT"i", &in, &in_length_arg, &out_length_arg)) return (NULL); + /* Check the size of length args and convert from Py_ssize_t to int */ + if (in_length_arg > INT_MAX) + return (PyErr_Format(decompression_exception, "Decompression error (Input length is larger than INT_MAX)")); + if (out_length_arg > INT_MAX) + return (PyErr_Format(decompression_exception, "Decompression error (Output length is larger than INT_MAX)")); + + in_length = Py_SAFE_DOWNCAST(in_length_arg, Py_ssize_t, int); + out_length = Py_SAFE_DOWNCAST(out_length_arg, Py_ssize_t, int); + /* Call the compression function */ status = decompress_packet(in, in_length, &out, &out_length); pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy