Python 虚拟机

x86 函数调用栈

py文件被编译为字节码指令序列之后，就通过 Python 虚拟机执行字节码。
python 代码到成为运行的进程，这之间经过了什么操作呢？
在 x86 的机器上可执行文件会维护函数调用栈来保证函数的有序执行，下图为出现函数调用时的栈帧情况。
caller 作为函数调用者，需要做的工作：

  • 函数参数入栈
  • return address 入栈，当前 PC + 4

callee 是被调用函数，需要做的工作：

  • 保存上一栈帧的 ebp，用于在函数调用结束后恢复 caller 的栈帧
  • 将 esp 寄存器值存到 ebp (将 ebp 理解为 esp 寄存器的快照)
  • esp 值减少用于划定栈帧的大小
  • 保存寄存器、局部变量的值

python虚拟机 - 图1
python 的虚拟机需要完成类似工作。

运行时环境

PyCodeObject

Python 源代码中的任何一个 Code Block 都会创建一个 CodeObject

  1. /* Bytecode object */
  2. typedef struct {
  3.     PyObject_HEAD
  4.     int co_argcount;            /* #arguments, except *args */
  5.     int co_posonlyargcount;     /* #positional only arguments */
  6.     int co_kwonlyargcount;      /* #keyword only arguments */
  7.     int co_nlocals;             /* #local variables */
  8.     int co_stacksize;           /* #entries needed for evaluation stack */
  9.     int co_flags;               /* CO_..., see below */
  10.     int co_firstlineno;         /* first source line number */
  11.     PyObject *co_code;          /* instruction opcodes */
  12.     PyObject *co_consts;        /* list (constants used) */
  13.     PyObject *co_names;         /* list of strings (names used) */
  14.     PyObject *co_varnames;      /* tuple of strings (local variable names) */
  15.     PyObject *co_freevars;      /* tuple of strings (free variable names) */
  16.     PyObject *co_cellvars;      /* tuple of strings (cell variable names) */
  17.     /* The rest aren't used in either hash or comparisons, except for co_name,
  18.        used in both. This is done to preserve the name and line number
  19.        for tracebacks and debuggers; otherwise, constant de-duplication
  20.        would collapse identical functions/lambdas defined on different lines.
  21.     */
  22.     Py_ssize_t *co_cell2arg;    /* Maps cell vars which are arguments. */
  23.     PyObject *co_filename;      /* unicode (where it was loaded from) */
  24.     PyObject *co_name;          /* unicode (name, for reference) */
  25.     PyObject *co_lnotab;        /* string (encoding addr<->lineno mapping) See
  26.                                    Objects/lnotab_notes.txt for details. */
  27.     void *co_zombieframe;       /* for optimization only (see frameobject.c) */
  28.     PyObject *co_weakreflist;   /* to support weakrefs to code objects */
  29.     /* Scratch space for extra data relating to the code object.
  30.        Type is a void* to keep the format private in codeobject.c to force
  31.        people to go through the proper APIs. */
  32.     void *co_extra;
  33.     /* Per opcodes just-in-time cache
  34.      *
  35.      * To reduce cache size, we use indirect mapping from opcode index to
  36.      * cache object:
  37.      *   cache = co_opcache[co_opcache_map[next_instr - first_instr] - 1]
  38.      */
  39.     // co_opcache_map is indexed by (next_instr - first_instr).
  40.     //  * 0 means there is no cache for this opcode.
  41.     //  * n > 0 means there is cache in co_opcache[n-1].
  42.     unsigned char *co_opcache_map;
  43.     _PyOpcache *co_opcache;
  44.     int co_opcache_flag;  // used to determine when create a cache.
  45.     unsigned char co_opcache_size;  // length of co_opcache.
  46. } PyCodeObject;

PyFrameObject

没发生一次函数调用都会创建一个执行环境，每发生一次函数调用都会创建一个新的执行环境，之间环境之间通过 f_back 连接起来形成链式结构的运行时环境（类比一个函数调用栈）。
每一个执行环境就是一个 PYFrameObject 的实例，改对象是一个变长的对象，里面包含 PyCodeObject、内置变量、全局变量、局部变量、栈顶、栈底这些信息。

  1. typedef struct _frame {
  2.     PyObject_VAR_HEAD
  3.     struct _frame *f_back;      /* previous frame, or NULL */
  4.     PyCodeObject *f_code;       /* code segment */
  5.     PyObject *f_builtins;       /* builtin symbol table (PyDictObject) */
  6.     PyObject *f_globals;        /* global symbol table (PyDictObject) */
  7.     PyObject *f_locals;         /* local symbol table (any mapping) */
  8.     PyObject **f_valuestack;    /* points after the last local */
  9.     /* Next free slot in f_valuestack.  Frame creation sets to f_valuestack.
  10.        Frame evaluation usually NULLs it, but a frame that yields sets it
  11.        to the current stack top. */
  12.     PyObject **f_stacktop;
  13.     PyObject *f_trace;          /* Trace function */
  14.     char f_trace_lines;         /* Emit per-line trace events? */
  15.     char f_trace_opcodes;       /* Emit per-opcode trace events? */
  16.     /* Borrowed reference to a generator, or NULL */
  17.     PyObject *f_gen;
  18.     int f_lasti;                /* Last instruction if called */
  19.     /* Call PyFrame_GetLineNumber() instead of reading this field
  20.        directly.  As of 2.3 f_lineno is only valid when tracing is
  21.        active (i.e. when f_trace is set).  At other times we use
  22.        PyCode_Addr2Line to calculate the line from the current
  23.        bytecode index. */
  24.     int f_lineno;               /* Current line number */
  25.     int f_iblock;               /* index in f_blockstack */
  26.     char f_executing;           /* whether the frame is still executing */
  27.     PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */
  28.     PyObject *f_localsplus[1];  /* locals+stack, dynamically sized */
  29. } PyFrameObject;

代码

字节码

  1. # 编译之后成为 PyCodeObject 存储在堆中
  2. >>> data = open('demo.py').read()
  3. >>> co = compile(data, "demo.py", "exec")
  4. >>> co       
  5. <code object <module> at 0x00000203E2EEA270, file "demo.py", line 1>
  6. # 访问co的局部变量、栈大小
  7. >>> type(co)     
  8. <class 'code'>
  9. >>> co.co_consts
  10. (<code object foo at 0x00000203E2EEA1E0, file "demo.py", line 1>, 'foo', '__main__', 100, None)
  11. >>> co.co_varnames
  12. ()
  13. >>> co.co_stacksize
  14. 2
  15. # 查看字节码
  16. >>> import dis     
  17. >>> dis.dis(co) 
  18.   1           0 LOAD_CONST               0 (<code object foo at 0x00000203E2EEA1E0, file "demo.py", line 1>)
  19.               2 LOAD_CONST               1 ('foo')
  20.               4 MAKE_FUNCTION            0
  21.               6 STORE_NAME               0 (foo)
  22.  13           8 LOAD_NAME                1 (__name__)
  23.              10 LOAD_CONST               2 ('__main__')
  24.              12 COMPARE_OP               2 (==)
  25.              14 POP_JUMP_IF_FALSE       42
  26.  14          16 LOAD_CONST               3 (100)
  27.              18 STORE_GLOBAL             2 (a)
  28.  15          20 LOAD_NAME                3 (print)
  29.              22 LOAD_GLOBAL              2 (a)
  30.              24 CALL_FUNCTION            1
  31.              26 POP_TOP
  32.  16          28 LOAD_NAME                0 (foo)
  33.              30 CALL_FUNCTION            0
  34.              32 POP_TOP
  35.  17          34 LOAD_NAME                3 (print)
  36.              36 LOAD_GLOBAL              2 (a)
  37.              38 CALL_FUNCTION            1
  38.              40 POP_TOP
  39.         >>   42 LOAD_CONST               4 (None)
  40.              44 RETURN_VALUE
  41. Disassembly of <code object foo at 0x00000203E2EEA1E0, file "demo.py", line 1>:
  42.   3           0 LOAD_CONST               1 (200)
  43.               2 STORE_GLOBAL             0 (a)
  44.   4           4 LOAD_CONST               2 (1000)
  45.               6 STORE_DEREF              0 (b)
  46.   5           8 LOAD_GLOBAL              1 (print)
  47.              10 LOAD_DEREF               0 (b)
  48.              12 CALL_FUNCTION            1
  49.              14 POP_TOP
  50.   7          16 LOAD_CLOSURE             0 (b)
  51.              18 BUILD_TUPLE              1
  52.              20 LOAD_CONST               3 (<code object bar at 0x00000203E2EEA4B0, file "demo.py", line 7>)
  53.              22 LOAD_CONST               4 ('foo.<locals>.bar')
  54.              24 MAKE_FUNCTION            8
  55.              26 STORE_FAST               0 (bar)
  56.  10          28 LOAD_FAST                0 (bar)
  57.              30 CALL_FUNCTION            0
  58.              32 POP_TOP
  59.  11          34 LOAD_GLOBAL              1 (print)
  60.              38 CALL_FUNCTION            1
  61.              40 POP_TOP
  62.              42 LOAD_CONST               0 (None)
  63.              44 RETURN_VALUE
  64. Disassembly of <code object bar at 0x00000203E2EEA4B0, file "demo.py", line 7>:
  65.   9           0 LOAD_CONST               1 (2000)
  66.               2 STORE_DEREF              0 (b)
  67.               4 LOAD_CONST               0 (None)
  68.               6 RETURN_VALUE
  69. >>>

变量作用域

  1. import inspect 
  2. def bar():
  3.     # nonlocal b
  4.     # b = 2000
  5.     # print(b)
  6.     frame = inspect.currentframe()
  7.     print(f"Callee fun: {frame.f_code.co_name}")
  8.     print(f"Callee's local: {frame.f_locals}")
  9.     print(f"Callee's global: {frame.f_globals}")
  10.     caller = frame.f_back
  11.     print(f"Caller fun: {caller.f_code.co_name}")
  12.     print(f"Caller's local: {caller.f_locals}")
  13.     print(f"Caller's global: {caller.f_globals}")
  14. def foo():
  15.     global a
  16.     a = 200
  17.     b = 1000
  18.     print(b)
  19.     bar()
  20.     print(b)
  21. if __name__ == "__main__":
  22.     a = 100
  23.     print(a)
  24.     foo()
  25.     print(a)

输出：

  1. 100
  2. 1000
  3. Callee fun: bar
  4. Callee's local: {'frame': <frame at 0x000002A0782775D8, file '.\\demo.py', line 10, code bar>}
  5. Callee's global: {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <_frozen_importlib_external.SourceFileLoader object at 0x000002A07890C208>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>, '__file__': '.\\demo.py', '__cached__': None, 'inspect': <module 'inspect' from 'C:\\Users\\Administrator\\AppData\\Local\\Programs\\Python\\Python37\\lib\\inspect.py'>, 'bar': <function bar at 0x000002A078949CA8>, 'foo': <function foo at 0x000002A0789CB288>, 'a': 200}
  6. Caller fun: foo
  7. Caller's local: {'b': 1000}
  8. Caller's global: {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <_frozen_importlib_external.SourceFileLoader object at 0x000002A07890C208>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>, '__file__': '.\\demo.py', '__cached__': None, 'inspect': <module 'inspect' from 'C:\\Users\\Administrator\\AppData\\Local\\Programs\\Python\\Python37\\lib\\inspect.py'>, 'bar': <function bar at 0x000002A078949CA8>, 'foo': <function foo at 0x000002A0789CB288>, 'a': 200}
  9. 1000
  10. 200

这里可以看到

  • 被调用函数能访问到调用者的栈帧信息
  • 全局变量被存储在每一个栈帧中

    内嵌函数

    把 bar 内嵌在 foo 函数内，并且两个函数有相同变量名称的局部变量
    1. import inspect 
    2. def foo():
    3.   global a
    4.   a = 200
    5.   b = 1000
    6.   print(b)
    7.   def bar():
    8.       b = 2000
    9.       print(b)
    10.       frame = inspect.currentframe()
    11.       print(f"Callee fun: {frame.f_code.co_name}")
    12.       print(f"Callee's local: {frame.f_locals}")
    13.       caller = frame.f_back
    14.       print(f"Caller fun: {caller.f_code.co_name}")
    15.       print(f"Caller's local: {caller.f_locals}")
    16.   bar()
    17.   print(b)
    18. if __name__ == "__main__":
    19.   a = 100
    20.   print(a)
    21.   foo()
    22.   print(a)
    我们能看到同名局部变量之间互不影响
    1. 100
    2. 1000
    3. 2000
    4. Callee fun: bar
    5. Callee's local: {'b': 2000, 'frame': <frame at 0x0000025051CEAAF8, file '.\\demo.py', line 13, code bar>}
    6. Caller fun: foo
    7. Caller's local: {'b': 1000, 'bar': <function foo.<locals>.bar at 0x000002505249A318>}
    8. 1000
    9. 200
    不定义这个同名局部变量，直接打印变量 b
    1. import inspect 
    2. def foo():
    3.   global a
    4.   a = 200
    5.   b = 1000
    6.   print(b)
    7.   def bar():
    8.       # nonlocal b  
    9.       print(b)
    10.       frame = inspect.currentframe()
    11.       print(f"Callee fun: {frame.f_code.co_name}")
    12.       print(f"Callee's local: {frame.f_locals}")
    13.       caller = frame.f_back
    14.       print(f"Caller fun: {caller.f_code.co_name}")
    15.       print(f"Caller's local: {caller.f_locals}")
    16.   bar()
    17.   print(b)
    18. if __name__ == "__main__":
    19.   a = 100
    20.   print(a)
    21.   foo()
    22.   print(a)
    此时程序不会报错，并且直接将上层函数的变量作为自己的局部变量，nonlocal b 写不写输出都一样。
    1. 100
    2. 1000
    3. 1000
    4. Callee fun: bar
    5. Callee's local: {'frame': <frame at 0x000002199FBD0E08, file '.\\demo.py', line 13, code bar>, 'b': 1000}
    6. Caller fun: foo
    7. Caller's local: {'bar': <function foo.<locals>.bar at 0x00000219A022A288>, 'b': 1000}
    8. 1000
    9. 200

    参考

    https://he11olx.com/2018/07/21/1.CPython3.6源码分析/1.5.Python Code Frame/
    《Python 源码剖析》