# -*- coding: utf-8 -*- import atexit import base64 import contextlib from io import StringIO import importlib import inspect import os import py_compile import re import shutil import sre_constants import subprocess import sys import tempfile import traceback from unittest import TestCase # get path to `tests` directory TESTS_DIR = os.path.dirname(__file__) # A state variable to determine if the test environment has been configured. _suite_configured = False # Temporary directory containing all files generated by this test process # Set during setUpSuite() _output_dir = '' def setUpSuite(): """Configure the entire test suite. This only needs to be run once, prior to the first test. """ global _output_dir global _suite_configured if _suite_configured: return def remove_output_dir(): global _output_dir if _output_dir != '': try: shutil.rmtree(_output_dir) except FileNotFoundError: pass atexit.register(remove_output_dir) _output_dir = tempfile.mkdtemp(dir=TESTS_DIR) if os.environ.get('PRECOMPILE', 'true').lower() == 'true': print("building 'batavia.js'") proc = subprocess.Popen( [os.path.join(os.path.dirname(TESTS_DIR), "node_modules", ".bin", "webpack"), "--bail"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, ) try: out, err = proc.communicate(timeout=60) except subprocess.TimeoutExpired: proc.kill() out, err = proc.communicate() raise if proc.returncode != 0: raise Exception("Error compiling batavia sources: " + out.decode('ascii')) else: print("Not precompiling 'batavia.js' as part of test run") _suite_configured = True @contextlib.contextmanager def capture_output(redirect_stderr=True): oldout, olderr = sys.stdout, sys.stderr try: out = StringIO() sys.stdout = out if redirect_stderr: sys.stderr = out else: sys.stderr = StringIO() yield out except: if redirect_stderr: traceback.print_exc() else: raise finally: sys.stdout, sys.stderr = oldout, olderr def adjust(text, run_in_function=False): """Adjust a code sample to remove leading whitespace.""" lines = text.split('\n') if len(lines) == 1: return text if lines[0].strip() == '': lines = lines[1:] first_line = lines[0].lstrip() n_spaces = len(lines[0]) - len(first_line) final_lines = [(' ' if run_in_function else '') + line[n_spaces:] for line in lines] if run_in_function: final_lines = [ "def test_function():", ] + final_lines + [ "test_function()", ] return '\n'.join(final_lines) def runAsPython(test_dir, main_code, extra_code=None, run_in_function=False, args=None): """Run a block of Python code with the Python interpreter.""" # Output source code into test directory with open(os.path.join(test_dir, 'test.py'), 'w', encoding='utf-8') as py_source: py_source.write(adjust(main_code, run_in_function=run_in_function)) if extra_code: for name, code in extra_code.items(): path = name.split('.') path[-1] = path[-1] + '.py' if len(path) != 1: try: os.makedirs(os.path.join(test_dir, *path[:-1])) except FileExistsError: pass with open(os.path.join(test_dir, *path), 'w', encoding="utf-8") as py_source: py_source.write(adjust(code)) if args is None: args = [] env_copy = os.environ.copy() env_copy['PYTHONIOENCODING'] = 'UTF-8' proc = subprocess.Popen( [sys.executable, "test.py"] + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=test_dir, env=env_copy, ) out = proc.communicate() return out[0].decode('utf8') JS_EXCEPTION = re.compile('Traceback \(most recent call last\):\r?\n( File "(?P.*)", line (?P\d+), in .*\r?\n)+(?P.*?): (?P.*\r?\n)') # NOQA JS_STACK = re.compile(' File "(?P.*)", line (?P\d+), in .*\r?\n') JS_BOOL_TRUE = re.compile('true') JS_BOOL_FALSE = re.compile('false') JS_FLOAT_DECIMAL = re.compile(r'''(?.*)", line (?P\d+), in .*\r?\n .*\r?\n)+(?P.*?): (?P.*\r?\n)') # NOQA PYTHON_STACK = re.compile(' File "(?P.*)", line (?P\d+), in .*\r?\n .*\r?\n') PYTHON_FLOAT_EXP = re.compile('(\d+)e(-)?0?(\d+)') PYTHON_NEGATIVE_ZERO_J = re.compile('-0j\)') # Prevent floating point discrepancies in very low significant digits from being an issue FLOAT_PRECISION = re.compile('(\\.\d{5})\d+') MEMORY_REFERENCE = re.compile('0x[\dABCDEFabcdef]{4,16}') def transforms(**transform_args): """ injects a JSCleaner and PYCleaner object into the function use this as a decarator to configure which transformations should be performed """ def _dec(function): def wrapper(self, *args, **kwargs): # js_cleaner js_excludes = ['py_test_script', 'py_str_excep'] js_params = { jsk : jsv for jsk, jsv in transform_args.items() if jsk not in js_excludes } js_cleaner = JSCleaner(**js_params) # py_cleaner py_excludes = ['js_bool', 'decimal', 'float_exp'] py_params = { pyk : pyv for pyk, pyv in transform_args.items() if pyk not in py_excludes } py_cleaner = PYCleaner(**py_params) res = function(self, js_cleaner, py_cleaner, *args, **kwargs) return res wrapper.__name__ = function.__name__ wrapper.__doc__ = function.__doc__ return wrapper return _dec class JSCleaner: def __init__(self, err_msg = True, memory_ref = True, js_bool = True, decimal = True, float_exp = True, complex_num = True, high_precision_float = True, test_ref = True, custom = True): self.transforms = { 'err_msg': err_msg, 'memory_ref': memory_ref, 'js_bool': js_bool, 'decimal': decimal, 'float_exp': float_exp, 'complex_num': complex_num, 'high_precision_float': high_precision_float, 'test_ref': test_ref, 'custom': custom } def cleanse(self, js_input, substitutions): """ cleanse output from javascript """ # Test the specific message out = js_input if self.transforms['err_msg']: out = JS_EXCEPTION.sub('### EXCEPTION ###{linesep}\\g: \\g'.format(linesep=os.linesep), js_input) stack = JS_STACK.findall(js_input) stacklines = [] test_dir = os.path.join(os.getcwd(), 'tests', 'temp') for filename, line in stack: if filename.startswith(test_dir): filename = filename[len(test_dir) + 1:] stacklines.append( " %s:%s" % ( filename, line ) ) out = '%s%s%s' % ( out, os.linesep.join(stacklines), os.linesep if stack else '' ) # Normalize memory references from output if self.transforms['memory_ref']: out = MEMORY_REFERENCE.sub("0xXXXXXXXX", out) if self.transforms['js_bool']: # Normalize true and false to True and False out = JS_BOOL_TRUE.sub("True", out) out = JS_BOOL_FALSE.sub("False", out) if self.transforms['decimal']: # Replace floating point numbers in decimal form with # the form used by python for match in JS_FLOAT_DECIMAL.findall(out): out = out.replace(match, str(float(match))) if self.transforms['float_exp']: # Format floating point numbers using a lower case e try: out = JS_FLOAT_EXP.sub('\\1e\\2\\3', out) except: pass if self.transforms['complex_num']: # Replace large integers in a complex number with floating point. for match in JS_LARGE_COMPLEX.findall(out): out = out.replace(match, str(float(match))) if self.transforms['high_precision_float']: # Replace high precision floats with abbreviated forms out = FLOAT_PRECISION.sub('\\1...', out) if self.transforms['test_ref']: # Replace references to the test script with something generic out = out.replace("'test.py'", '***EXECUTABLE***') if self.transforms['custom']: # Replace all the explicit data substitutions if substitutions: for to_value, from_values in substitutions.items(): for from_value in from_values: # check for regex if hasattr(from_value, 'pattern'): out = re.sub(from_value.pattern, re.escape(to_value), out, 0, re.MULTILINE) else: out = out.replace(from_value, to_value) out = out.replace('\r\n', '\n') # trim trailing whitespace on non-blank lines out = '\n'.join(o.rstrip() for o in out.split('\n')) return out class PYCleaner: def __init__(self, err_msg = True, memory_ref = True, float_exp = True, complex_num = True, high_precision_float = True, test_ref = True, custom = True): self.transforms = { 'err_msg': err_msg, 'memory_ref': memory_ref, 'float_exp': float_exp, 'complex_num': complex_num, 'high_precision_float': high_precision_float, 'test_ref': test_ref, 'custom': custom } def cleanse(self, py_input, substitutions): """ cleanse output from python """ out = py_input if self.transforms['err_msg']: # Test the specific message out = PYTHON_EXCEPTION.sub( '### EXCEPTION ###{linesep}\\g: \\g'.format(linesep=os.linesep), py_input ) stack = PYTHON_STACK.findall(py_input) out = '%s%s%s' % ( out, os.linesep.join( [ " %s:%s" % (s[0], s[1]) for s in stack ] ), os.linesep if stack else '' ) if self.transforms['memory_ref']: # Normalize memory references from output out = MEMORY_REFERENCE.sub("0xXXXXXXXX", out) if self.transforms['float_exp']: # Format floating point numbers using a lower case e try: out = PYTHON_FLOAT_EXP.sub('\\1e\\2\\3', out) except sre_constants.error: pass if self.transforms['complex_num']: # Replace "-0j" with "+0j" out = PYTHON_NEGATIVE_ZERO_J.sub('+0j)', out) if self.transforms['high_precision_float']: # Replace high precision floats with abbreviated forms out = FLOAT_PRECISION.sub('\\1...', out) if self.transforms['test_ref']: # Replace references to the test script with something generic out = out.replace("'test.py'", '***EXECUTABLE***') # Python 3.4.4 changed the message describing strings in exceptions out = out.replace( 'argument must be a string or', 'argument must be a string, a bytes-like object or' ) if self.transforms['custom']: if substitutions: for to_value, from_values in substitutions.items(): for from_value in from_values: # check for regex if hasattr(from_value, 'pattern'): out = re.sub(from_value.pattern, re.escape(to_value), out, 0, re.MULTILINE) else: out = out.replace(from_value, to_value) out = out.replace('\r\n', '\n') # trim trailing whitespace on non-blank lines out = '\n'.join(o.rstrip() for o in out.split('\n')) return out class TranspileTestCase(TestCase): @classmethod def setUpClass(cls): global _output_dir setUpSuite() cls.temp_dir = _output_dir def assertCodeExecution( self, code, message=None, extra_code=None, run_in_global=True, run_in_function=True, args=None, substitutions=None, js_cleaner=JSCleaner(), py_cleaner=PYCleaner()): "Run code as native python, and under JavaScript and check the output is identical" self.maxDiff = None # ================================================== # Pass 1 - run the code in the global context # ================================================== if run_in_global: try: self.makeTempDir() # Run the code as Python and as JavaScript. py_out = runAsPython( self.temp_dir, code, extra_code=extra_code, run_in_function=False, args=args ) js_out = self.runAsJavaScript( code, extra_code=extra_code, run_in_function=False, args=args, python_exists=True ) except Exception as e: self.fail(e) finally: # Clean up the test directory where the class file was written. shutil.rmtree(self.temp_dir) # Cleanse the Python and JavaScript output, producing a simple # normalized format for exceptions, floats etc. js_out = js_cleaner.cleanse(js_out, substitutions) py_out = py_cleaner.cleanse(py_out, substitutions) # Confirm that the output of the JavaScript code is the same as the Python code. if message: context = 'Global context: %s' % message else: context = 'Global context' self.assertEqual(js_out, py_out, context) # ================================================== # Pass 2 - run the code in a function's context # ================================================== if run_in_function: try: self.makeTempDir() # Run the code as Python and as Java. py_out = runAsPython( self.temp_dir, code, extra_code=extra_code, run_in_function=True, args=args ) js_out = self.runAsJavaScript( code, extra_code=extra_code, run_in_function=True, args=args, python_exists=True ) except Exception as e: self.fail(e) finally: # Clean up the test directory where the class file was written. shutil.rmtree(self.temp_dir) # Cleanse the Python and JavaScript output, producing a simple # normalized format for exceptions, floats etc. js_out = js_cleaner.cleanse(js_out, substitutions) py_out = py_cleaner.cleanse(py_out, substitutions) # Confirm that the output of the JavaScript code is the same as the Python code. if message: context = 'Function context: %s' % message else: context = 'Function context' self.assertEqual(js_out, py_out, context) def assertJavaScriptExecution( self, code, out, extra_code=None, js=None, run_in_global=True, run_in_function=True, args=None, substitutions=None, same=True, js_cleaner=JSCleaner()): "Run code under JavaScript and check the output is as expected" self.maxDiff = None # ================================================== # Prep - compile any required JavaScript sources # ================================================== # Cleanse the Python output, producing a simple # normalized format for exceptions, floats etc. py_out = adjust(out) # ================================================== # Pass 1 - run the code in the global context # ================================================== if run_in_global: try: self.makeTempDir() # Run the code as Javascript. js_out = self.runAsJavaScript( code, extra_code=extra_code, js=js, run_in_function=False, args=args ) except Exception as e: self.fail(e) finally: # Clean up the test directory where the class file was written. shutil.rmtree(self.temp_dir) # Cleanse the JavaScript output, producing a simple # normalized format for exceptions, floats etc. js_out = js_cleaner.cleanse(js_out, substitutions) # Compare the output of the JavaScript code with the Python code. if same: self.assertEqual(js_out, py_out, 'Global context') else: self.assertNotEqual(js_out, py_out, 'Global context') # ================================================== # Pass 2 - run the code in a function's context # ================================================== if run_in_function: try: self.makeTempDir() # Run the code as JavaScript. js_out = self.runAsJavaScript( code, extra_code=extra_code, js=js, run_in_function=True, args=args ) except Exception as e: self.fail(e) finally: # Clean up the test directory where the class file was written. shutil.rmtree(self.temp_dir) # Cleanse the JavaScript output, producing a simple # normalized format for exceptions, floats etc. js_out = js_cleaner.cleanse(js_out, substitutions) # Compare the output of the JavaScript code with the Python code. if same: self.assertEqual(js_out, py_out, 'Function context') else: self.assertNotEqual(js_out, py_out, 'Function context') def makeTempDir(self): """Create a "temp" subdirectory in the class's generated temporary directory if it doesn't currently exist.""" try: os.mkdir(self.temp_dir) except FileExistsError: pass def runAsJavaScript( self, main_code, extra_code=None, js=None, run_in_function=False, args=None, python_exists=False ): # Output source code into test directory assert isinstance(main_code, (str, bytes)), ( 'I have no idea how to run tests for code of type {}' ''.format(type(main_code)) ) if not python_exists: if isinstance(main_code, str): py_filename = os.path.join(self.temp_dir, 'test.py') with open(py_filename, 'w', encoding='utf-8') as py_source: py_source.write(adjust(main_code, run_in_function=run_in_function)) modules = [] # Temporarily move into the test directory. cwd = os.getcwd() os.chdir(self.temp_dir) if isinstance(main_code, str): py_compile.compile('test.py') with open(importlib.util.cache_from_source('test.py'), 'rb') as compiled: modules.append(('test', base64.encodebytes(compiled.read()), 'test.py')) elif isinstance(main_code, bytes): modules.append(('test', main_code, 'test.py')) if extra_code: for name, code in extra_code.items(): path = name.split('.') path[-1] = path[-1] + '.py' py_filename = os.path.join(*path) if not python_exists: if len(path) != 1: try: os.makedirs(os.path.join(self.temp_dir, *path[:-1])) except FileExistsError: pass with open(py_filename, 'w') as py_source: py_source.write(adjust(code)) py_compile.compile(py_filename) with open(importlib.util.cache_from_source(py_filename), 'rb') as compiled: modules.append((name, base64.encodebytes(compiled.read()), py_filename)) if args is None: args = [] # Convert the dictionary of modules into a payload payload = [] for name, code, filename in modules: lines = code.decode('utf-8').split('\n') output = '"%s"' % '" +\n "'.join(line for line in lines if line) if name.endswith('.__init__'): name = name.rsplit('.', 1)[0] payload.append( ' "%s": {\n' % name + ' "__python__": true,\n' + ' "bytecode": %s,\n' % output + ' "filename": "%s"\n' % filename + ' }' ) if js: for name, code in js.items(): payload.append( ' "%s": {\n' % name + ' "javascript": %s\n' % name + ' }' ) with open(os.path.join(self.temp_dir, 'test.js'), 'w') as js_file: js_file.write(adjust(""" var batavia = require('../../dist/batavia.js'); %s var modules = { %s }; var vm = new batavia.VirtualMachine({ loader: function(name) { var payload = modules[name]; if (payload === undefined) { return null; } return payload; }, frame: null }); vm.run('test', []); """) % ( '\n'.join( adjust(code) for name, code in sorted(js.items()) ) if js else '', ',\n'.join(payload) ) ) proc = subprocess.Popen( ['node', 'test.js'] + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, # cwd=self.temp_dir, ) out = proc.communicate() # Move back to the old current working directory. os.chdir(cwd) return out[0].decode('utf8') class NotImplementedToExpectedFailure: def run(self, result=None): # Override the run method to inject the "expectingFailure" marker # when the test case runs. if self._testMethodName in getattr(self, 'not_implemented', []): # Mark 'expecting failure' on class. It will only be applicable # for this specific run. method = getattr(self, self._testMethodName) def wrapper(*args, **kwargs): method(*args, **kwargs) wrapper.__unittest_expecting_failure__ = True setattr(self, self._testMethodName, wrapper) return super().run(result=result) SAMPLE_DATA = { 'bool': [ 'True', 'False', ], 'bytearray': [ # 'bytearray()', # 'bytearray(1)', # 'bytearray([1, 2, 3])', 'bytearray(b"hello world")', ], 'bytes': [ 'b""', 'b"This is another string of bytes"', 'b"\x01\x75"', # mixing nonprintable and printable bytes ], 'class': [ 'type(1)', 'type("a")', # 'type(object())', # TODO: re-enable this when object() is implemented 'type("MyClass", (object,), {})', ], 'complex': [ '1j', '3.14159265j', '1+2j', '3-4j', # '-5j', ], 'dict': [ '{}', '{"a": 1, "c": 2.3456, "d": "another"}', '{"a": {"a": 1}, "b": {"b": 2}}', ], 'float': [ '2.3456', '0.0', '-3.14159', '-4.81756', '5.5', '-3.5', '4.5', '-4.5', ], 'frozenset': [ 'frozenset()', 'frozenset([1])', # 'frozenset({"1"})', this reveals some bugs in our code 'frozenset({1, 2.3456, "another"})', ], 'int': [ '3', '0', '-5', '-3', '5', '1', '-1', '9223372036854775807', '9223372036854775808', '-9223372036854775807', '-9223372036854775808', '18446744073709551615', '18446744073709551616', '18446744073709551617', '-18446744073709551615', '-18446744073709551616', '-18446744073709551617', '1361129467683753853853498429727072845824', '-1361129467683753853853498429727072845824', '179769313486231590772930519078902473361797697894230657273430081157732675805500963132708477322407536021120113879871393357658789768814416622492847430639474124377767893424865485276302219601246094119453082952085005768838150682342462881473913110540827237163350510684586298239947245938479716304835356329624224137216', # NOQA '-179769313486231590772930519078902473361797697894230657273430081157732675805500963132708477322407536021120113879871393357658789768814416622492847430639474124377767893424865485276302219601246094119453082952085005768838150682342462881473913110540827237163350510684586298239947245938479716304835356329624224137216', # NOQA ], 'list': [ '[]', '[1]', '[3, 4, 5]', '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]', '["a","b","c"]', '[[1, 2], [3, 4]]', ], 'range': [ 'range(0)', 'range(5)', 'range(2, 7)', 'range(2, 7, 2)', 'range(7, 2, -1)', 'range(7, 2, -2)', ], 'set': [ 'set()', '{1, 2.3456, "another"}', ], 'slice': [ 'slice(0)', 'slice(5)', 'slice(2, 7)', 'slice(2, 7, 2)', 'slice(7, 2, -1)', 'slice(7, 2, -2)', ], 'str': [ '""', '"3"', '"This is another string"', '"Mÿ hôvèrçràft îß fûłl öf éêlś"', '"One arg: %s"', '"Three args: %s | %s | %s"', ], 'tuple': [ '()', '(False,)', '(1,)', '(1, 2)', '(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)', '(3, 1.2, True, )', '(1, 2.3456, "another")', '((1, 2), (3, 4))', '((3, 4), (1, 2))' ], 'None': [ 'None', ], 'NotImplemented': [ 'NotImplemented', ], } SAMPLE_SUBSTITUTIONS = { # Normalize set ordering "{1, 2.3456, 'another'}": [ "{1, 'another', 2.3456}", "{2.3456, 1, 'another'}", "{2.3456, 'another', 1}", "{'another', 1, 2.3456}", "{'another', 2.3456, 1}", ], "{2.3456, 'another'}": [ "{'another', 2.3456}", ], "{'1': 1, 1: 2}": [ "{1: 2, '1': 1}", ], "{'1', 1}": [ "{1, '1'}", ], "{'a', 'b', 'c'}": [ "{'a', 'c', 'b'}", "{'b', 'a', 'c'}", "{'b', 'c', 'a'}", "{'c', 'a', 'b'}", "{'c', 'b', 'a'}", ], "{3, 4, 5}": [ "{3, 5, 4}", "{4, 3, 5}", "{4, 5, 3}", "{5, 3, 4}", "{5, 4, 3}", ], "{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}": [ # otherwise we have to generate 3628800 permutations re.compile(r'\{(?:[1-9]|10)(?:, (?:[1-9]|10)){9}\}'), ], # Normalize dictionary ordering "{'a': 1, 'c': 2.3456, 'd': 'another'}": [ "{'a': 1, 'd': 'another', 'c': 2.3456}", "{'c': 2.3456, 'd': 'another', 'a': 1}", "{'c': 2.3456, 'a': 1, 'd': 'another'}", "{'d': 'another', 'a': 1, 'c': 2.3456}", "{'d': 'another', 'c': 2.3456, 'a': 1}", ], "{'a': {'a': 1}, 'b': {'b': 2}}": [ "{'b': {'b': 2}, 'a': {'a': 1}}", ], #created after dict() call on nested set and list "{1: 2, 3: 4}":[ "{3: 4, 1: 2}", ], #created after list() call on dict "['a', 'b']" : [ "['b', 'a']", ], #created after tuple() call on dict "('a', 'b')" : [ "('b', 'a')", ], # Normalize set to list ordering "[1, 2.3456, 'another']": [ "[1, 'another', 2.3456]", "[2.3456, 1, 'another']", "['another', 1, 2.3456]", "['another', 2.3456, 1]", "[2.3456, 'another', 1]", ], # Normalize set to tuple ordering "(1, 2.3456, 'another')": [ "(1, 'another', 2.3456)", "(2.3456, 1, 'another')", "('another', 1, 2.3456)", "('another', 2.3456, 1)", "(2.3456, 'another', 1)", ], # Normalize dictionary keys to list ordering "['c', 'd', 'a']": [ "['a', 'c', 'd']", "['a', 'd', 'c']", "['c', 'a', 'd']", "['c', 'd', 'a']", "['d', 'a', 'c']", "['d', 'c', 'a']", ], # Normalize dictionary keys to tuple ordering "('c', 'd', 'a')": [ "('a', 'c', 'd')", "('a', 'd', 'c')", "('c', 'a', 'd')", "('c', 'd', 'a')", "('d', 'a', 'c')", "('d', 'c', 'a')", ], } def _unary_test(test_name, operation): def func(self): self.assertUnaryOperation( x_values=SAMPLE_DATA[self.data_type], operation=operation, format=self.format, substitutions=getattr(self, 'substitutions', SAMPLE_SUBSTITUTIONS) ) return func class UnaryOperationTestCase(NotImplementedToExpectedFailure): format = '' def assertUnaryOperation(self, x_values, operation, format, substitutions): self.assertCodeExecution( '##################################################\n'.join( adjust(""" try: print('>>> x = %(x)s') print('>>> %(format)s%(operation)sx') x = %(x)s print(%(format)s%(operation)sx) except Exception as e: print(type(e), ':', e) print() """ % { 'x': x, 'operation': operation, 'format': format, } ) for x in x_values ), "Error running %s" % operation, substitutions=substitutions, run_in_function=False, ) test_unary_positive = _unary_test('test_unary_positive', '+') test_unary_negative = _unary_test('test_unary_negative', '-') test_unary_not = _unary_test('test_unary_not', 'not ') test_unary_invert = _unary_test('test_unary_invert', '~') def _binary_test(test_name, operation, examples, small_ints=False): def func(self): # CPython will attempt to malloc itself to death for some operations, # e.g., 1 << (2**32) # so we have this dirty hack actuals = examples if small_ints and test_name.endswith('_int'): actuals = [x for x in examples if abs(int(x)) < 8192] self.assertBinaryOperation( x_values=SAMPLE_DATA[self.data_type], y_values=actuals, operation=operation, format=self.format, substitutions=getattr(self, 'substitutions', SAMPLE_SUBSTITUTIONS) ) return func class BinaryOperationTestCase(NotImplementedToExpectedFailure): format = '' y = 3 def assertBinaryOperation(self, x_values, y_values, operation, format, substitutions): data = [] for x in x_values: for y in y_values: data.append((x, y)) self.assertCodeExecution( '##################################################\n'.join( adjust(""" try: print('>>> x = %(x)s') print('>>> y = %(y)s') print('>>> %(format)s%(operation)s') x = %(x)s y = %(y)s print(%(format)s%(operation)s) except Exception as e: print(type(e), ':', e) print() """ % { 'x': x, 'y': y, 'operation': operation, 'format': format, } ) for x, y in data ), "Error running %s" % operation, substitutions=substitutions, run_in_function=False, ) for datatype, examples in SAMPLE_DATA.items(): vars()['test_add_%s' % datatype] = _binary_test( 'test_add_%s' % datatype, 'x + y', examples ) vars()['test_subtract_%s' % datatype] = _binary_test( 'test_subtract_%s' % datatype, 'x - y', examples ) vars()['test_multiply_%s' % datatype] = _binary_test( 'test_multiply_%s' % datatype, 'x * y', examples, small_ints=True ) vars()['test_floor_divide_%s' % datatype] = _binary_test( 'test_floor_divide_%s' % datatype, 'x // y', examples ) vars()['test_true_divide_%s' % datatype] = _binary_test( 'test_true_divide_%s' % datatype, 'x / y', examples ) vars()['test_modulo_%s' % datatype] = _binary_test( 'test_modulo_%s' % datatype, 'x % y', examples ) vars()['test_power_%s' % datatype] = _binary_test( 'test_power_%s' % datatype, 'x ** y', examples, small_ints=True ) vars()['test_subscr_%s' % datatype] = _binary_test( 'test_subscr_%s' % datatype, 'x[y]', examples ) vars()['test_lshift_%s' % datatype] = _binary_test( 'test_lshift_%s' % datatype, 'x << y', examples, small_ints=True ) vars()['test_rshift_%s' % datatype] = _binary_test( 'test_rshift_%s' % datatype, 'x >> y', examples, small_ints=True ) vars()['test_and_%s' % datatype] = _binary_test( 'test_and_%s' % datatype, 'x & y', examples ) vars()['test_xor_%s' % datatype] = _binary_test( 'test_xor_%s' % datatype, 'x ^ y', examples ) vars()['test_or_%s' % datatype] = _binary_test( 'test_or_%s' % datatype, 'x | y', examples ) vars()['test_lt_%s' % datatype] = _binary_test( 'test_lt_%s' % datatype, 'x < y', examples ) vars()['test_le_%s' % datatype] = _binary_test( 'test_le_%s' % datatype, 'x <= y', examples ) vars()['test_gt_%s' % datatype] = _binary_test( 'test_gt_%s' % datatype, 'x > y', examples ) vars()['test_ge_%s' % datatype] = _binary_test( 'test_ge_%s' % datatype, 'x >= y', examples ) vars()['test_eq_%s' % datatype] = _binary_test( 'test_eq_%s' % datatype, 'x == y', examples ) vars()['test_ne_%s' % datatype] = _binary_test( 'test_ne_%s' % datatype, 'x != y', examples ) def _inplace_test(test_name, operation, examples, small_ints=False): def func(self): actuals = examples if small_ints and test_name.endswith('_int'): actuals = [x for x in examples if abs(int(x)) < 8192] self.assertInplaceOperation( x_values=SAMPLE_DATA[self.data_type], y_values=actuals, operation=operation, format=self.format, substitutions=getattr(self, 'substitutions', SAMPLE_SUBSTITUTIONS) ) return func class InplaceOperationTestCase(NotImplementedToExpectedFailure): format = '' y = 3 def assertInplaceOperation(self, x_values, y_values, operation, format, substitutions): data = [] for x in x_values: for y in y_values: data.append((x, y)) self.assertCodeExecution( '##################################################\n'.join( adjust(""" try: print('>>> x = %(x)s') print('>>> y = %(y)s') print('>>> %(operation)s') print('>>> %(format)sx') x = %(x)s y = %(y)s %(operation)s print(%(format)sx) except Exception as e: print(type(e), ':', e) print() """ % { 'x': x, 'y': y, 'operation': operation, 'format': format, } ) for x, y in data ), "Error running %s" % operation, substitutions=substitutions, run_in_function=False, ) for datatype, examples in SAMPLE_DATA.items(): vars()['test_add_%s' % datatype] = _inplace_test( 'test_add_%s' % datatype, 'x += y', examples ) vars()['test_subtract_%s' % datatype] = _inplace_test( 'test_subtract_%s' % datatype, 'x -= y', examples ) vars()['test_multiply_%s' % datatype] = _inplace_test( 'test_multiply_%s' % datatype, 'x *= y', examples, small_ints=True ) vars()['test_floor_divide_%s' % datatype] = _inplace_test( 'test_floor_divide_%s' % datatype, 'x //= y', examples ) vars()['test_true_divide_%s' % datatype] = _inplace_test( 'test_true_divide_%s' % datatype, 'x /= y', examples ) vars()['test_modulo_%s' % datatype] = _inplace_test( 'test_modulo_%s' % datatype, 'x %= y', examples ) vars()['test_power_%s' % datatype] = _inplace_test( 'test_power_%s' % datatype, 'x **= y', examples, small_ints=True ) vars()['test_lshift_%s' % datatype] = _inplace_test( 'test_lshift_%s' % datatype, 'x <<= y', examples, small_ints=True ) vars()['test_rshift_%s' % datatype] = _inplace_test( 'test_rshift_%s' % datatype, 'x >>= y', examples ) vars()['test_and_%s' % datatype] = _inplace_test( 'test_and_%s' % datatype, 'x &= y', examples ) vars()['test_xor_%s' % datatype] = _inplace_test( 'test_xor_%s' % datatype, 'x ^= y', examples ) vars()['test_or_%s' % datatype] = _inplace_test( 'test_or_%s' % datatype, 'x |= y', examples ) def _builtin_test(test_name, operation, examples, small_ints=False): def func(self): # bytes() gives implementation-dependent errors for sizes > 2**64, # we'll skip testing with those values rather than cargo-culting # the exact same exceptions actuals = examples if self.small_ints and test_name.endswith('_int'): actuals = [x for x in examples if abs(int(x)) < 8192] self.assertBuiltinFunction( x_values=actuals, f_values=self.functions, operation=operation, format=self.format, substitutions=getattr(self, 'substitutions', SAMPLE_SUBSTITUTIONS) ) return func class BuiltinFunctionTestCase(NotImplementedToExpectedFailure): format = '' substitutions = SAMPLE_SUBSTITUTIONS small_ints = False def assertBuiltinFunction(self, f_values, x_values, operation, format, substitutions): data = [] for f in f_values: for x in x_values: data.append((f, x)) self.assertCodeExecution( '##################################################\n'.join( adjust(""" try: print('>>> f = %(f)s') print('>>> x = %(x)s') print('>>> %(format)s%(operation)s') f = %(f)s x = %(x)s print(%(format)s%(operation)s) except Exception as e: print(type(e), ':', e) print() """ % { 'f': f, 'x': x, 'operation': operation, 'format': format, } ) for f, x in data ), "Error running %s" % operation, substitutions=substitutions, run_in_function=False, ) for datatype, examples in SAMPLE_DATA.items(): vars()['test_%s' % datatype] = _builtin_test('test_%s' % datatype, 'f(x)', examples) vars()['test_noargs'] = _builtin_test('test_noargs', 'f()', examples=['"_noargs (not used)"']) def _builtin_twoarg_test(test_name, operation, examples1, examples2): def func(self): self.assertBuiltinTwoargFunction( f_values=self.functions, x_values=examples1, y_values=examples2, operation=operation, format=self.format, substitutions=getattr(self, 'substitutions', SAMPLE_SUBSTITUTIONS) ) return func class BuiltinTwoargFunctionTestCase(NotImplementedToExpectedFailure): format = '' def assertBuiltinTwoargFunction(self, f_values, x_values, y_values, operation, format, substitutions): data = [] for f in f_values: for x in x_values: for y in y_values: data.append((f, x, y)) # filter out very large integers for some operations so as not # to crash CPython data = [(f, x, y) for f, x, y in data if not (f == 'pow' and x.lstrip('-').isdigit() and y.lstrip('-').isdigit() and (abs(int(x)) > 8192 or abs(int(y)) > 8192))] self.assertCodeExecution( '##################################################\n'.join( adjust(""" try: print('>>> f = %(f)s') print('>>> x = %(x)s') print('>>> y = %(y)s') print('>>> %(format)s%(operation)s') f = %(f)s x = %(x)s y = %(y)s print(%(format)s%(operation)s) except Exception as e: print(type(e), ':', e) print() """ % { 'f': f, 'x': x, 'y': y, 'operation': operation, 'format': format, } ) for f, x, y in data ), "Error running %s" % operation, substitutions=substitutions, run_in_function=False, ) for datatype1, examples1 in SAMPLE_DATA.items(): for datatype2, examples2 in SAMPLE_DATA.items(): vars()['test_%s_%s' % (datatype1, datatype2)] = _builtin_twoarg_test( 'test_%s_%s' % (datatype1, datatype2), 'f(x, y)', examples1, examples2 ) def _module_one_arg_func_test(name, module, f, examples, small_ints=False): # Factorials can make us run out of memory and crash. # so we have this dirty hack actuals = examples if small_ints and name.endswith('_int'): actuals = [x for x in examples if abs(int(x)) < 8192] def func(self): self.assertOneArgModuleFuction( name=name, module=module, func=f, x_values=actuals, substitutions=getattr(self, 'substitutions', SAMPLE_SUBSTITUTIONS) ) return func def _module_two_arg_func_test(name, module, f, examples, examples2): def func(self): self.assertTwoArgModuleFuction( name=name, module=module, func=f, x_values=examples, y_values=examples2, substitutions=getattr(self, 'substitutions', SAMPLE_SUBSTITUTIONS) ) return func numerics = {'bool', 'float', 'int'} class ModuleFunctionTestCase(NotImplementedToExpectedFailure): numerics_only = False def assertOneArgModuleFuction(self, name, module, func, x_values, substitutions): self.assertCodeExecution( '##################################################\n'.join( adjust(""" try: print('>>> import %(m)s') print('>>> f = %(m)s.%(f)s') print('>>> x = %(x)s') print('>>> f(x)') import %(m)s f = %(m)s.%(f)s x = %(x)s print(f(x)) except Exception as e: print(type(e), ':', e) print() """ % { 'f': func, 'x': x, 'm': module, } ) for x in x_values ), "Error running %s module %s" % (module, name), substitutions=substitutions, run_in_function=False, ) def assertTwoArgModuleFuction(self, name, module, func, x_values, y_values, substitutions): self.assertCodeExecution( '##################################################\n'.join( adjust(""" try: print('>>> import %(m)s') print('>>> f = %(m)s.%(f)s') print('>>> x = %(x)s') print('>>> y = %(y)s') print('>>> f(x, y)') import %(m)s f = %(m)s.%(f)s x = %(x)s y = %(y)s print(f(x, y)) except Exception as e: print(type(e), ':', e) print() """ % { 'f': func, 'x': x, 'y': y, 'm': module, } ) for x in x_values for y in y_values ), "Error running %s module %s" % (module, name), substitutions=substitutions, run_in_function=False, ) @classmethod def add_one_arg_tests(klass, module, functions, numerics_only=False): for func in functions: for datatype, examples in SAMPLE_DATA.items(): if numerics_only and datatype not in numerics: continue name = 'test_%s_%s_%s' % (module, func, datatype) small_ints = module == 'math' and func == 'factorial' setattr(klass, name, _module_one_arg_func_test(name, 'math', func, examples, small_ints=small_ints)) @classmethod def add_two_arg_tests(klass, module, functions, numerics_only=False): for func in functions: for datatype, examples in SAMPLE_DATA.items(): if numerics_only and datatype not in numerics: continue for datatype2, examples2 in SAMPLE_DATA.items(): if numerics_only and datatype2 not in numerics: continue name = 'test_%s_%s_%s_%s' % (module, func, datatype, datatype2) setattr(klass, name, _module_two_arg_func_test(name, 'math', func, examples, examples2))