第9章 性能优化技术*

import time

def fun():
    sum_value = 0
    for i in range(1000000):
        sum_value += i

start = time.perf_counter()
fun()
time.sleep(1)
end = time.perf_counter()
print('perf_counter:', end - start)

perf_counter: 1.0736476539996147

start = time.process_time()
fun()
time.sleep(1)
end = time.process_time()
print('process_time:', end - start)  # process_time中不包含time.sleep(1)造成的程序休眠时间

process_time: 0.07273000000000018

$ python -m timeit '"-".join(str(n) for n in range(100))'
10000 loops, best of 5: 23.5 usec per loop

$ python -m timeit -n 2000 -r 10 '"-".join(str(n) for n in range(100))'
2000 loops, best of 10: 23.5 usec per loop

import timeit
timeit.timeit('"-".join(str(n) for n in range(100))', number=10000)

0.2476290219992734

timeit.repeat('"-".join(str(n) for n in range(100))', number=10000, repeat=5)

[0.2479564710001796,
 0.2286210049996953,
 0.2277683940001225,
 0.24715779699999985,
 0.23120317899974907]

import timeit

def bubble_sort(lst):
    """冒泡排序"""
    for i in range(len(lst)-1, 0, -1):
        for j in range(0, i):
            if lst[j] > lst[j + 1]:
                lst[j], lst[j+1] = lst[j+1], lst[j]
    return lst

t1 = timeit.timeit('bubble_sort(lst)',
                  setup='''from __main__ import bubble_sort; import random; lst = list(range(1000)); random.shuffle(lst)''',
                  number=10)

t2 = timeit.repeat('bubble_sort(lst)',
                  setup='''from __main__ import bubble_sort; import random; lst = list(range(1000)); random.shuffle(lst)''',
                  number=10,
                  repeat=3)
print(t1)
print(t2)

0.5808773019998625
[0.5691154360001747, 0.5654969030001666, 0.5676186800001233]

def fib(n):
    if n == 0:
        return 0
    elif n == 1:
        return 1
    else:
        return fib(n - 1) + fib(n - 2)

def fib_list(n):
    seq = []
    if n > 0:
        seq.extend(fib_list(n - 1))
    seq.append(fib(n))
    return seq

if __name__ == '__main__':
    fib_list(30)

$ python -m cProfile test_profile.py

         7049218 function calls (96 primitive calls) in 2.015 seconds

   Ordered by: standard name

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
        1    0.000    0.000    2.015    2.015 test_profile.py:1(<module>)
7049123/31    2.014    0.000    2.014    0.065 test_profile.py:1(fib)
     31/1    0.000    0.000    2.015    2.015 test_profile.py:10(fib_list)
        1    0.000    0.000    2.015    2.015 {built-in method builtins.exec}
       31    0.000    0.000    0.000    0.000 {method 'append' of 'list' objects}
        1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}
       30    0.000    0.000    0.000    0.000 {method 'extend' of 'list' objects}

def fib(n):
    if n == 0:
        return 0
    elif n == 1:
        return 1
    else:
        return fib(n - 1) + fib(n - 2)

def fib_list(n):
    seq = []
    if n > 0:
        seq.extend(fib_list(n - 1))
    seq.append(fib(n))
    return seq

import cProfile
cProfile.run('fib_list(30)')

         7049218 function calls (96 primitive calls) in 1.998 seconds

   Ordered by: standard name

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
7049123/31    1.997    0.000    1.997    0.064 <ipython-input-6-3823c2297ba6>:1(fib)
     31/1    0.000    0.000    1.998    1.998 <ipython-input-6-3823c2297ba6>:9(fib_list)
        1    0.000    0.000    1.998    1.998 <string>:1(<module>)
        1    0.000    0.000    1.998    1.998 {built-in method builtins.exec}
       31    0.000    0.000    0.000    0.000 {method 'append' of 'list' objects}
        1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}
       30    0.000    0.000    0.000    0.000 {method 'extend' of 'list' objects}

import cProfile
p = cProfile.Profile()
p.enable()
fib_list(30)
p.disable()
p.print_stats(sort='tottime')

         7049260 function calls (138 primitive calls) in 2.040 seconds

   Ordered by: internal time

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
7049123/31    2.040    0.000    2.040    0.066 <ipython-input-6-3823c2297ba6>:1(fib)
     31/1    0.000    0.000    2.040    2.040 <ipython-input-6-3823c2297ba6>:9(fib_list)
        2    0.000    0.000    0.000    0.000 {built-in method builtins.compile}
        2    0.000    0.000    2.040    1.020 interactiveshell.py:3293(run_code)
       30    0.000    0.000    0.000    0.000 {method 'extend' of 'list' objects}
       31    0.000    0.000    0.000    0.000 {method 'append' of 'list' objects}
        2    0.000    0.000    0.000    0.000 codeop.py:135(__call__)
        2    0.000    0.000    0.000    0.000 contextlib.py:82(__init__)
        4    0.000    0.000    0.000    0.000 compilerop.py:138(extra_flags)
        1    0.000    0.000    0.000    0.000 <ipython-input-7-9649c2a4d5ce>:5(<module>)
        2    0.000    0.000    0.000    0.000 hooks.py:103(__call__)
        2    0.000    0.000    0.000    0.000 contextlib.py:117(__exit__)
        2    0.000    0.000    0.000    0.000 contextlib.py:108(__enter__)
        4    0.000    0.000    0.000    0.000 {built-in method builtins.getattr}
        2    0.000    0.000    2.040    1.020 {built-in method builtins.exec}
        2    0.000    0.000    0.000    0.000 traitlets.py:545(__get__)
        2    0.000    0.000    0.000    0.000 contextlib.py:238(helper)
        2    0.000    0.000    0.000    0.000 traitlets.py:526(get)
        1    0.000    0.000    2.040    2.040 <ipython-input-7-9649c2a4d5ce>:4(<module>)
        2    0.000    0.000    0.000    0.000 interactiveshell.py:3231(compare)
        4    0.000    0.000    0.000    0.000 {built-in method builtins.next}
        2    0.000    0.000    0.000    0.000 ipstruct.py:125(__getattr__)
        2    0.000    0.000    0.000    0.000 interactiveshell.py:1276(user_global_ns)
        2    0.000    0.000    0.000    0.000 hooks.py:168(pre_run_code_hook)
        1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}

import pstats
p = pstats.Stats('result.out')
p.strip_dirs().sort_stats('name').print_stats()
print('-'*100)
p.sort_stats('cumulative').print_stats()
print('-'*100)
p.sort_stats('time').print_stats()

---------------------------------------------------------------------------

FileNotFoundError                         Traceback (most recent call last)

<ipython-input-8-85ff513ed0d6> in <module>
      1 import pstats
----> 2 p = pstats.Stats('result.out')
      3 p.strip_dirs().sort_stats('name').print_stats()
      4 print('-'*100)
      5 p.sort_stats('cumulative').print_stats()


~/anaconda3/envs/lesson/lib/python3.8/pstats.py in __init__(self, stream, *args)
     94             arg = args[0]
     95             args = args[1:]
---> 96         self.init(arg)
     97         self.add(*args)
     98


~/anaconda3/envs/lesson/lib/python3.8/pstats.py in init(self, arg)
    108         self.stats = {}
    109         self.sort_arg_dict = {}
--> 110         self.load_stats(arg)
    111         try:
    112             self.get_top_level_stats()


~/anaconda3/envs/lesson/lib/python3.8/pstats.py in load_stats(self, arg)
    121             return
    122         elif isinstance(arg, str):
--> 123             with open(arg, 'rb') as f:
    124                 self.stats = marshal.load(f)
    125             try:


FileNotFoundError: [Errno 2] No such file or directory: 'result.out'

# pypy_test.py
import timeit
def bubble_sort(lst):
    """冒泡排序"""
    for i in range(len(lst)-1, 0, -1):
        for j in range(0, i):
            if lst[j] > lst[j + 1]:
                lst[j], lst[j+1] = lst[j+1], lst[j]
    return lst

t = timeit.timeit('bubble_sort(lst)',
                  setup='''from __main__ import bubble_sort; \
                  import random; lst = list(range(1000)); random.shuffle(lst)''',
                  number=10)
print(t)

0.5926239299997178

$ python pypy_test.py 
0.571357171
$
$ pypy3 pypy_test.py 
0.022235398006159812

import timeit
from numba import jit

def factorial(n):
    '''阶乘'''
    fac = 1
    for i in range(1, n+1):
        fac = fac*i
    return fac

@jit
def factorial_jit(n):
    fac = 1
    for i in range(1, n+1):
        fac = fac*i
    return fac

timeit.timeit('factorial(10000)', setup='from __main__ import factorial', number=100)

2.0420013419998213

timeit.timeit('factorial_jit(10000)', setup='from __main__ import factorial_jit', number=100)

7.113244510999721

from numba import jit, int32

@jit(int32(int32, int32))
def f(x, y):
    return x + y

f(2**31, 2**31 + 1)

1

@jit(nopython=True)
def f(x, y):
    return x + y

@jit(cache=True)
def f(x, y):
    return x + y

// 文件add.c
double add(double x, double y) {
    return x + y;
}

import ctypes
lib = ctypes.cdll.LoadLibrary('./libadd.so')
add = lib.add
add.argtypes = (ctypes.c_double, ctypes.c_double)  # 参数的数据类型
add.restype = ctypes.c_double                      # 返回值的数据类型
print(add(1.0, 2))

// divide.c
int divide(int a, int b, int *remainder) {
    int quot = a / b;
    *remainder = a % b;
    return quot;
}

gcc -o libdivide.so -shared -fPIC divide.c

import ctypes
_lib = ctypes.cdll.LoadLibrary('libdivide.so')
_divide = _lib.divide
_divide.argtypes = (ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int))
_divide.restype = ctypes.c_int

def divide(x, y):
    rem = ctypes.c_int()
    quot = _divide(x, y, rem)
    return quot, rem.value

>>> divide(42, 8)
(5, 2)

// avg.c
double avg(double *a, int n) {
    int i;
    double total = 0.0;
    for (i = 0; i < n; i++) {
        total += a[i];
    }
    return total / n;
}

gcc -o libavg.so -shared -fPIC avg.c

import ctypes
_lib = ctypes.cdll.LoadLibrary('libavg.so')

class DoubleArray:
    def from_param(self, param):
        return ((ctypes.c_double)*len(param))(*param)

array = DoubleArray()
_avg = _lib.avg
_avg.argtypes = (array, ctypes.c_int)
_avg.restype = ctypes.c_double

def avg(values):
    return _avg(values, len(values))

>>> avg([1, 2, 3, 4, 5, 6])
3.5

// dist.c

#include <math.h>

typedef struct Point {
    double x,y;
} Point;

double distance(Point *p1, Point *p2) {
    return hypot(p1->x - p2->x, p1->y - p2->y);
}

gcc -o libdist.so -shared -fPIC dist.c

import ctypes
_lib = ctypes.cdll.LoadLibrary('libdist.so')

class Point(ctypes.Structure):
    _fields_ = [('x', ctypes.c_double),
                ('y', ctypes.c_double)]

distance = _lib.distance
distance.argtypes = (ctypes.POINTER(Point), ctypes.POINTER(Point))
distance.restype = ctypes.c_double

>>> p1 = Point(1, 2)
>>> p2 = Point(4, 5)
>>> distance(p1, p2)
4.242640687119285

// rectangle.cpp

class Rectangle
{
public:
    Rectangle(float, float);
    double area();
    double perimeter();

private:
    float width_;
    float height_;
};

Rectangle::Rectangle(float width, float height)
{
    width_ = width;
    height_ = height;
}

double Rectangle::area()
{
    return width_ * height_;
}

double Rectangle::perimeter()
{
    return 2 * width_ + 2 * height_;
}

// 以C的方式编译如下函数
extern "C"
{
    Rectangle *Rectangle_new(double width, double height)
    {
        return new Rectangle(width, height);
    }
    double area(Rectangle *rect)
    {
        return rect->area();
    }
    double perimeter(Rectangle *rect)
    {
        return rect->perimeter();
    }
}

g++ -o librectangle.so -shared -fPIC rectangle.cpp

import ctypes

_lib = ctypes.cdll.LoadLibrary('librectangle.so')

class Rectangle:
    def __init__(self, width, height):
        self._methods = dict()
        self._methods['area'] = _lib.area
        self._methods['perimeter'] = _lib.perimeter

        _lib.Rectangle_new.argtypes = (ctypes.c_double, ctypes.c_double)
        _lib.Rectangle_new.restype = ctypes.c_void_p

        _lib.area.argtypes = (ctypes.c_void_p,)
        _lib.area.restype = ctypes.c_double

        _lib.perimeter.argtypes = (ctypes.c_void_p,)
        _lib.perimeter.restype = ctypes.c_double

        self.obj = _lib.Rectangle_new(width, height)
        self._m_name = None

    def __getattr__(self, attr):
        self._m_name = attr
        return self.__call_method

    def __call_method(self, *args):
        return self._methods[self._m_name](self.obj, *args)

>>> rect = Rectangle(3, 5)
>>> rect.area()
15.0
>>> rect.perimeter()
16.0

// (1) Python参数列表包含两个整数
int x, y;
PyArg_ParseTuple(args, "ii", &x, &y);

// (2) Python参数列表包含三个参数，分别为字符串、整数和浮点数
const char* x;
int y;
double z;
PyArg_ParseTuple(args, "sid", &x, &y, &z)

// (2) Python参数为列表
PyObject *seq;
PyArg_ParseTuple(args, "O", &seq);
seq = PySequence_List(seq);
int seqlen = PySequence_Length(seq);

// testlib.c
#include <math.h>

// 最大公约数
int gcd(int x, int y) {
    int g = y;
    while (x > 0) {
        g = x;
        x = y % x;
        y = g;
    }
    return g;
}

// 除法
int divide(int a, int b, int *remainder) {
    int quot = a / b;
    *remainder = a % b;
    return quot;
}

// 列表求均值
double avg(double *a, int n) {
    int i;
    double total = 0.0;
    for (i = 0; i < n; i++) {
        total += a[i];
    }
    return total / n;
}

// 结构体
typedef struct Point {
    double x,y;
} Point;

// 结构体参数
double distance(Point *p1, Point *p2) {
    return hypot(p1->x - p2->x, p1->y - p2->y);
}

gcc -o testlib.o -c testlib.c

// testlib.h

#include <math.h>

extern int gcd(int, int);
extern int divide(int a, int b, int *remainder);
extern double avg(double *a, int n);

typedef struct Point {
    double x,y;
} Point;

extern double distance(Point *p1, Point *p2);

// testlibpy.c

#include "Python.h"
#include "testlib.h"

// 函数
static PyObject *py_gcd(PyObject *self, PyObject *args)
{
    int x, y, result;

    if (!PyArg_ParseTuple(args, "ii", &x, &y))
    {
        return NULL;
    }
    result = gcd(x, y);
    return Py_BuildValue("i", result);
}

// 指针作为参数
static PyObject *py_divide(PyObject *self, PyObject *args)
{
    int a, b, quotient, remainder;
    if (!PyArg_ParseTuple(args, "ii", &a, &b))
    {
        return NULL;
    }
    quotient = divide(a, b, &remainder);
    return Py_BuildValue("(ii)", quotient, remainder);
}

// 列表作为参数
static PyObject *py_avg(PyObject *self, PyObject *args)
{
    PyObject *seq;
    double *dbar, result;
    int seqlen;

    // 获取序列参数
    if (!PyArg_ParseTuple(args, "O", &seq))
        return 0;
    seq = PySequence_List(seq);

    // 将序列复制为double数组
    seqlen = PySequence_Length(seq);
    dbar = malloc(seqlen * sizeof(double));
    for (int i = 0; i < seqlen; i++)
    {
        PyObject *item = PyList_GetItem(seq, i);
        dbar[i] = PyFloat_AsDouble(PyNumber_Float(item));
    }

    // 释放空间，计算并返回结果
    Py_DECREF(seq);
    result = avg(dbar, seqlen);
    free(dbar);
    return Py_BuildValue("d", result);
}

// Point 对象的 Destructor 函数
static void del_Point(PyObject *obj)
{
    free(PyCapsule_GetPointer(obj, "Point"));
}

// 结构体：创建 Point 对象
static PyObject *py_Point(PyObject *self, PyObject *args)
{
    Point *p;
    double x, y;
    if (!PyArg_ParseTuple(args, "dd", &x, &y))
    {
        return NULL;
    }
    p = (Point *)malloc(sizeof(Point));
    p->x = x;
    p->y = y;

    // 创建capsule。capsule类似于指针
    return PyCapsule_New(p, "Point", del_Point);
}

// 结构体作为参数
static PyObject *py_distance(PyObject *self, PyObject *args)
{
    Point *p1, *p2;
    PyObject *py_p1, *py_p2;
    double result;

    if (!PyArg_ParseTuple(args, "OO", &py_p1, &py_p2))
    {
        return NULL;
    }

    // 提取capsule中的指针
    p1 = (Point *)PyCapsule_GetPointer(py_p1, "Point");
    p2 = (Point *)PyCapsule_GetPointer(py_p2, "Point");

    result = distance(p1, p2);
    return Py_BuildValue("d", result);
}

// 模块函数列表
static PyMethodDef methods[] = {
    {"gcd", py_gcd, METH_VARARGS, "greatest common divisor"},
    {"divide", py_divide, METH_VARARGS, "integer division"},
    {"avg", py_avg, METH_VARARGS, "list average"},
    {"distance", py_distance, METH_VARARGS, "point distance"},
    {"Point", py_Point, METH_VARARGS, "point"},
    {NULL, NULL, 0, NULL}};

// 配置模块
static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT,
    "testlib",        // 模块名
    "A lib for test", // 模块文档字符串
    -1,               // 解释器状态大小，-1表示用全局变量保存状态
    methods};

// 模块初始化函数
PyMODINIT_FUNC PyInit_testlib(void)
{
    return PyModule_Create(&module);
}

# setup.py
from distutils.core import setup, Extension

setup(name='testlib',
      ext_modules=[
          Extension('testlib',
                    ['testlibpy.c'],
                    include_dirs=['/path/to/python/env/include/python3.8'],
                    library_dirs=['/path/to/file/testlib.o/'],
                    libraries=['testlib.o']
                    )
      ]
      )

python setup.py build_ext --inplace

>>> from testlib import *
>>> gcd(35,42)
7
>>> divide(42, 8)
(5, 2)
>>> avg([1, 2, 3, 4, 5, 6])
3.5
>>> p1 = Point(1, 2)
>>> p2 = Point(4, 5)
>>> distance(p1, p2)
4.242640687119285

# 文件 testlibpy/functions.py
import numpy as np

def add(x, y):
    return x + y

def average(lst):
    return np.average(lst)

class Point:
    def __init__(self, x, y):
        self.x, self.y = x, y

def distance(p1, p2):
    return np.sqrt((p1.x - p2.x)**2 + (p1.y - p2.y)**2)

# 文件 testlibpy/__init__.py
from .functions import *

# 文件 setup.py
from setuptools import setup, find_packages

desc = '工具包的简要说明'
long_description = open('./README.md').read()

setup(
    name="testlibpy",
    version="0.0.1",
    author="pystudy",
    author_email="xxxxxx@xxxxxx.com",
    license='MIT',
    description=desc,
    long_description=long_description,
    long_description_content_type='text/markdown',
    url="https://xxxxxx.com/testlibpy",
    classifiers=[ 'Development Status :: 3 - Alpha',
                  'Programming Language :: Python :: 3',
                  'Operating System :: OS Independent'],
    packages=find_packages(include=['testlibpy']), 
    install_requires=['numpy > 1.15'],
    python_requires='>=3.6',
)