Merge pull request #233 from pyutils/fix/malformed_co_code

Patch for duplicate functions

Author: Erotemic

CHANGELOG.rst

@@ -3,11 +3,14 @@ Changes
 
 4.1.0
 ~~~~
+* FIX: skipzeros now checks for zero hits instead of zero time
+* FIX: Fixed errors in Python 3.11 with duplicate functions.
 * FIX: ``show_text`` now increases column sizes or switches to scientific notation to maintain alignment
 * ENH: ``show_text`` now has new options: sort and summarize
 * ENH: Added new CLI arguments ``-srm`` to ``line_profiler`` to control sorting, rich printing, and summary printing.
 * ENH: New global ``profile`` function that can be enabled by ``--profile`` or ``LINE_PROFILE=1``.
 * ENH: New auto-profile feature in ``kernprof`` that will profile all functions in specified modules.
+* ENH: Kernprof now outputs instructions on how to view results.
 * ENH: Added readthedocs integration: https://kernprof.readthedocs.io/en/latest/index.html
 
 4.0.3

clean.sh

@@ -19,6 +19,13 @@ rm -rf CMakeTmp
 rm -rf CMakeFiles
 rm -rf tests/htmlcov
 
+rm -rf demo_primes*
+rm -rf docs/demo.py*
+rm -rf docs/script_to_profile.py*
+rm -rf tests/complex_example.py.lprof
+rm -rf tests/complex_example.py.prof
+rm -rf script_to_profile.py*
+
 
 if [ -f "distutils.errors" ]; then
     rm distutils.errors || echo "skip rm"

line_profiler/_line_profiler.pyx

@@ -4,6 +4,7 @@ This is the Cython backend used in :py:mod:`line_profiler.line_profiler`.
 """
 from .python25 cimport PyFrameObject, PyObject, PyStringObject
 from sys import byteorder
+import sys
 cimport cython
 from cpython.version cimport PY_VERSION_HEX
 from libc.stdint cimport int64_t
@@ -106,8 +107,8 @@ cdef inline int64 compute_line_hash(uint64 block_hash, uint64 linenum):
     return block_hash ^ linenum
 
 def label(code):
-    """ Return a (filename, first_lineno, func_name) tuple for a given code
-    object.
+    """ 
+    Return a (filename, first_lineno, func_name) tuple for a given code object.
 
     This is the same labelling as used by the cProfile module in Python 2.5.
     """
@@ -142,17 +143,19 @@ cpdef _code_replace(func, co_code):
 
 # Note: this is a regular Python class to allow easy pickling.
 class LineStats(object):
-    """ Object to encapsulate line-profile statistics.
-
-    Attributes
-    ----------
-    timings : dict
-        Mapping from (filename, first_lineno, function_name) of the profiled
-        function to a list of (lineno, nhits, total_time) tuples for each
-        profiled line. total_time is an integer in the native units of the
-        timer.
-    unit : float
-        The number of seconds per timer unit.
+    """ 
+    Object to encapsulate line-profile statistics.
+
+    Attributes:
+
+        timings (dict):
+            Mapping from (filename, first_lineno, function_name) of the
+            profiled function to a list of (lineno, nhits, total_time) tuples
+            for each profiled line. total_time is an integer in the native
+            units of the timer.
+
+        unit (float):
+            The number of seconds per timer unit.
     """
     def __init__(self, timings, unit):
         self.timings = timings
@@ -225,7 +228,24 @@ cdef class LineProfiler:
         if code.co_code in self.dupes_map:
             self.dupes_map[code.co_code] += [code]
             # code hash already exists, so there must be a duplicate function. add no-op
-            co_code = code.co_code + (9).to_bytes(1, byteorder=byteorder) * (len(self.dupes_map[code.co_code]))
+            # co_code = code.co_code + (9).to_bytes(1, byteorder=byteorder) * (len(self.dupes_map[code.co_code]))
+
+            """
+            # Code to lookup the NOP opcode, which we will just hard code here
+            # instead of looking it up. Perhaps do a global lookup in the
+            # future.
+            NOP_VALUE: int = opcode.opmap['NOP']
+            """
+            NOP_VALUE: int = 9
+            # Op code should be 2 bytes as stated in
+            # https://docs.python.org/3/library/dis.html
+            # if sys.version_info[0:2] >= (3, 11):
+            NOP_BYTES = NOP_VALUE.to_bytes(2, byteorder=byteorder)
+            # else:
+            #     NOP_BYTES = NOP_VALUE.to_bytes(1, byteorder=byteorder)
+
+            co_padding = NOP_BYTES * (len(self.dupes_map[code.co_code]) + 1)
+            co_code = code.co_code + co_padding
             CodeType = type(code)
             code = _code_replace(func, co_code=co_code)
             try:
@@ -333,7 +353,8 @@ cdef class LineProfiler:
         unset_trace()
 
     def get_stats(self):
-        """ Return a LineStats object containing the timings.
+        """ 
+        Return a LineStats object containing the timings.
         """
         cdef dict cmap
 
@@ -373,7 +394,8 @@ cdef class LineProfiler:
 @cython.wraparound(False)
 cdef int python_trace_callback(object self_, PyFrameObject *py_frame, int what,
 PyObject *arg):
-    """ The PyEval_SetTrace() callback.
+    """ 
+    The PyEval_SetTrace() callback.
     """
     cdef LineProfiler self
     cdef object code

line_profiler/autoprofile/ast_profle_transformer.py

@@ -78,7 +78,13 @@ def visit_FunctionDef(self, node):
             (_ast.FunctionDef): node
                 function/method with profiling decorator
         """
-        if self._profiler_name not in (d.id for d in node.decorator_list):
+        decor_ids = set()
+        for decor in node.decorator_list:
+            try:
+                decor_ids.add(decor.id)
+            except AttributeError:
+                ...
+        if self._profiler_name not in decor_ids:
             node.decorator_list.append(ast.Name(id=self._profiler_name, ctx=ast.Load()))
         return self.generic_visit(node)
 

line_profiler/explicit_profiler.py

@@ -376,7 +376,21 @@ def show(self):
             self._profile.dump_stats(lprof_output_fpath)
             print('Wrote profile results to %s' % lprof_output_fpath)
             print('To view details run:')
-            print(sys.executable + ' -m line_profiler -rtmz ' + str(lprof_output_fpath))
+            py_exe = _python_command()
+            print(py_exe + ' -m line_profiler -rtmz ' + str(lprof_output_fpath))
+
+
+def _python_command():
+    """
+    Return a command that corresponds to :py:obj:`sys.executable`.
+    """
+    import shutil
+    if shutil.which('python') == sys.executable:
+        return 'python'
+    elif shutil.which('python3') == sys.executable:
+        return 'python3'
+    else:
+        return sys.executable
 
 
 # Construct the global profiler.

line_profiler/line_profiler.py

@@ -286,8 +286,10 @@ def show_func(filename, start_lineno, func_name, timings, unit,
     if stream is None:
         stream = sys.stdout
 
+    total_hits = sum(t[1] for t in timings)
     total_time = sum(t[2] for t in timings)
-    if stripzeros and total_time == 0:
+
+    if stripzeros and total_hits == 0:
         return
 
     if rich:

tests/complex_example.py

@@ -97,8 +97,6 @@ def fib_only_called_by_process(n):
     a, b = 0, 1
     while a < n:
         a, b = b, a + b
-    # FIXME: having two functions with the EXACT same code can cause issues
-    a = 'no longer exactly the same'
 
 
 @profile

tests/test_autoprofile.py

@@ -83,6 +83,55 @@ def func4(a):
     temp_dpath.delete()
 
 
+def test_duplicate_function_autoprofile():
+    """
+    Test that every function in a file is profiled when autoprofile is enabled.
+    """
+    temp_dpath = ub.Path(tempfile.mkdtemp())
+
+    code = ub.codeblock(
+        '''
+        def func1(a):
+            return a + 1
+
+        def func2(a):
+            return a + 1
+
+        def func3(a):
+            return a + 1
+
+        def func4(a):
+            return a + 1
+
+        func1(1)
+        func2(1)
+        func3(1)
+        ''')
+    with ub.ChDir(temp_dpath):
+
+        script_fpath = ub.Path('script.py')
+        script_fpath.write_text(code)
+
+        args = [sys.executable, '-m', 'kernprof', '-p', 'script.py', '-l', os.fspath(script_fpath)]
+        proc = ub.cmd(args)
+        print(proc.stdout)
+        print(proc.stderr)
+        proc.check_returncode()
+
+        args = [sys.executable, '-m', 'line_profiler', os.fspath(script_fpath) + '.lprof']
+        proc = ub.cmd(args)
+        raw_output = proc.stdout
+        print(raw_output)
+        proc.check_returncode()
+
+    assert 'Function: func1' in raw_output
+    assert 'Function: func2' in raw_output
+    assert 'Function: func3' in raw_output
+    assert 'Function: func4' in raw_output
+
+    temp_dpath.delete()
+
+
 def _write_demo_module(temp_dpath):
     """
     Make a dummy test module structure
@@ -258,10 +307,10 @@ def test_autoprofile_script_with_prof_imports():
     temp_dpath = ub.Path(tempfile.mkdtemp())
     script_fpath = _write_demo_module(temp_dpath)
 
-    import sys
-    if sys.version_info[0:2] >= (3, 11):
-        import pytest
-        pytest.skip('Failing due to the noop bug')
+    # import sys
+    # if sys.version_info[0:2] >= (3, 11):
+    #     import pytest
+    #     pytest.skip('Failing due to the noop bug')
 
     args = [sys.executable, '-m', 'kernprof', '--prof-imports', '-p', 'script.py', '-l', os.fspath(script_fpath)]
     proc = ub.cmd(args, cwd=temp_dpath, verbose=2)

tests/test_complex_case.py

@@ -64,30 +64,52 @@ def test_varied_complex_invocations():
                     'outpath': outpath,
                 })
 
+    # Add case for auto-profile
+    # FIXME: this runs, but doesn't quite work.
+    cases.append({
+        'runner': 'kernprof',
+        'kern_flags': '-l --prof-mod complex_example.py',
+        'env_line_profile': '0',
+        'profile_type': 'none',
+        'outpath': 'complex_example.py.lprof',
+        'ignore_checks': True,
+    })
+
+    if 0:
+        # FIXME: this does not run with prof-imports
+        cases.append({
+            'runner': 'kernprof',
+            'kern_flags': '-l --prof-imports --prof-mod complex_example.py',
+            'env_line_profile': '0',
+            'profile_type': 'none',
+            'outpath': 'complex_example.py.lprof',
+        })
+
     complex_fpath = get_complex_example_fpath()
 
     results = []
 
-    for item in cases:
+    for case in cases:
         temp_dpath = tempfile.mkdtemp()
         with ub.ChDir(temp_dpath):
             env = {}
 
-            outpath = item['outpath']
+            outpath = case['outpath']
             if outpath:
                 outpath = ub.Path(outpath)
 
             # Construct the invocation for each case
-            if item['runner'] == 'kernprof':
-                kern_flags = item['kern_flags']
+            if case['runner'] == 'kernprof':
+                kern_flags = case['kern_flags']
+                # FIXME:
                 # Note: kernprof doesn't seem to play well with multiprocessing
                 prog_flags = ' --process_size=0'
                 runner = f'{sys.executable} -m kernprof {kern_flags}'
             else:
-                env['LINE_PROFILE'] = item["env_line_profile"]
+                env['LINE_PROFILE'] = case["env_line_profile"]
                 runner = f'{sys.executable}'
                 prog_flags = ''
-            env['PROFILE_TYPE'] = item["profile_type"]
+            env['PROFILE_TYPE'] = case["profile_type"]
             command = f'{runner} {complex_fpath}' + prog_flags
 
             HAS_SHELL = LINUX
@@ -101,7 +123,7 @@ def test_varied_complex_invocations():
 
             info.check_returncode()
 
-            result = item.copy()
+            result = case.copy()
             if outpath:
                 result['outsize'] = outpath.stat().st_size
             else:
@@ -120,6 +142,7 @@ def test_varied_complex_invocations():
                 rich.print(table)
 
             # Ensure the scripts that produced output produced non-trivial output
-            for result in results:
-                if result['outpath'] is not None:
-                    assert result['outsize'] > 100
+            if not case.get('ignore_checks', False):
+                for result in results:
+                    if result['outpath'] is not None:
+                        assert result['outsize'] > 100

tests/test_duplicate_functions.py

@@ -0,0 +1,20 @@
+def test_duplicate_function():
+    """
+    Test from https://github.com/pyutils/line_profiler/issues/232
+    """
+    import line_profiler
+
+    class C:
+        def f1(self):
+            pass
+
+        def f2(self):
+            pass
+
+        def f3(self):
+            pass
+
+    profile = line_profiler.LineProfiler()
+    profile(C.f1)
+    profile(C.f2)
+    profile(C.f3)