Speed Up Your Python Program With Concurrency 介绍了 Python 中可用的基本并发类型(threading,asyncio,multiprocessing),以及在 IO 密集型或 CPU 密集型的应用;An Intro to Threading in Python 给出了一种线程安全的编程结构——使用队列的生产者与消费者。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
import concurrent.futures
import logging
import queue
import random
import threading
import time

def producer(queue, event):
"""Pretend we're getting a number from the network."""
while not event.is_set():
message = random.randint(1, 101)
logging.info("Producer got message: %s", message)
queue.put(message)

logging.info("Producer received event. Exiting")

def consumer(queue, event):
"""Pretend we're saving a number in the database."""
while not event.is_set() or not queue.empty():
message = queue.get()
logging.info(
"Consumer storing message: %s (size=%d)", message, queue.qsize()
)

logging.info("Consumer received event. Exiting")

if __name__ == "__main__":
format = "%(asctime)s: %(message)s"
logging.basicConfig(format=format, level=logging.INFO,
datefmt="%H:%M:%S")

pipeline = queue.Queue(maxsize=10)
event = threading.Event()
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
executor.submit(producer, pipeline, event)
executor.submit(consumer, pipeline, event)

time.sleep(0.1)
logging.info("Main: about to set event")
event.set()