1 | """An adaptation of Py2.3/2.4's Queue module which supports reentrant |
---|
2 | behavior, using RLock instead of Lock for its mutex object. |
---|
3 | |
---|
4 | This is to support the connection pool's usage of weakref callbacks to return |
---|
5 | connections to the underlying Queue, which can apparently in extremely |
---|
6 | rare cases be invoked within the ``get()`` method of the Queue itself, |
---|
7 | producing a ``put()`` inside the ``get()`` and therefore a reentrant |
---|
8 | condition.""" |
---|
9 | |
---|
10 | from collections import deque |
---|
11 | from time import time as _time |
---|
12 | from sqlalchemy.util import threading |
---|
13 | |
---|
14 | __all__ = ['Empty', 'Full', 'Queue'] |
---|
15 | |
---|
16 | class Empty(Exception): |
---|
17 | "Exception raised by Queue.get(block=0)/get_nowait()." |
---|
18 | |
---|
19 | pass |
---|
20 | |
---|
21 | class Full(Exception): |
---|
22 | "Exception raised by Queue.put(block=0)/put_nowait()." |
---|
23 | |
---|
24 | pass |
---|
25 | |
---|
26 | class Queue: |
---|
27 | def __init__(self, maxsize=0): |
---|
28 | """Initialize a queue object with a given maximum size. |
---|
29 | |
---|
30 | If `maxsize` is <= 0, the queue size is infinite. |
---|
31 | """ |
---|
32 | |
---|
33 | self._init(maxsize) |
---|
34 | # mutex must be held whenever the queue is mutating. All methods |
---|
35 | # that acquire mutex must release it before returning. mutex |
---|
36 | # is shared between the two conditions, so acquiring and |
---|
37 | # releasing the conditions also acquires and releases mutex. |
---|
38 | self.mutex = threading.RLock() |
---|
39 | # Notify not_empty whenever an item is added to the queue; a |
---|
40 | # thread waiting to get is notified then. |
---|
41 | self.not_empty = threading.Condition(self.mutex) |
---|
42 | # Notify not_full whenever an item is removed from the queue; |
---|
43 | # a thread waiting to put is notified then. |
---|
44 | self.not_full = threading.Condition(self.mutex) |
---|
45 | |
---|
46 | def qsize(self): |
---|
47 | """Return the approximate size of the queue (not reliable!).""" |
---|
48 | |
---|
49 | self.mutex.acquire() |
---|
50 | n = self._qsize() |
---|
51 | self.mutex.release() |
---|
52 | return n |
---|
53 | |
---|
54 | def empty(self): |
---|
55 | """Return True if the queue is empty, False otherwise (not reliable!).""" |
---|
56 | |
---|
57 | self.mutex.acquire() |
---|
58 | n = self._empty() |
---|
59 | self.mutex.release() |
---|
60 | return n |
---|
61 | |
---|
62 | def full(self): |
---|
63 | """Return True if the queue is full, False otherwise (not reliable!).""" |
---|
64 | |
---|
65 | self.mutex.acquire() |
---|
66 | n = self._full() |
---|
67 | self.mutex.release() |
---|
68 | return n |
---|
69 | |
---|
70 | def put(self, item, block=True, timeout=None): |
---|
71 | """Put an item into the queue. |
---|
72 | |
---|
73 | If optional args `block` is True and `timeout` is None (the |
---|
74 | default), block if necessary until a free slot is |
---|
75 | available. If `timeout` is a positive number, it blocks at |
---|
76 | most `timeout` seconds and raises the ``Full`` exception if no |
---|
77 | free slot was available within that time. Otherwise (`block` |
---|
78 | is false), put an item on the queue if a free slot is |
---|
79 | immediately available, else raise the ``Full`` exception |
---|
80 | (`timeout` is ignored in that case). |
---|
81 | """ |
---|
82 | |
---|
83 | self.not_full.acquire() |
---|
84 | try: |
---|
85 | if not block: |
---|
86 | if self._full(): |
---|
87 | raise Full |
---|
88 | elif timeout is None: |
---|
89 | while self._full(): |
---|
90 | self.not_full.wait() |
---|
91 | else: |
---|
92 | if timeout < 0: |
---|
93 | raise ValueError("'timeout' must be a positive number") |
---|
94 | endtime = _time() + timeout |
---|
95 | while self._full(): |
---|
96 | remaining = endtime - _time() |
---|
97 | if remaining <= 0.0: |
---|
98 | raise Full |
---|
99 | self.not_full.wait(remaining) |
---|
100 | self._put(item) |
---|
101 | self.not_empty.notify() |
---|
102 | finally: |
---|
103 | self.not_full.release() |
---|
104 | |
---|
105 | def put_nowait(self, item): |
---|
106 | """Put an item into the queue without blocking. |
---|
107 | |
---|
108 | Only enqueue the item if a free slot is immediately available. |
---|
109 | Otherwise raise the ``Full`` exception. |
---|
110 | """ |
---|
111 | return self.put(item, False) |
---|
112 | |
---|
113 | def get(self, block=True, timeout=None): |
---|
114 | """Remove and return an item from the queue. |
---|
115 | |
---|
116 | If optional args `block` is True and `timeout` is None (the |
---|
117 | default), block if necessary until an item is available. If |
---|
118 | `timeout` is a positive number, it blocks at most `timeout` |
---|
119 | seconds and raises the ``Empty`` exception if no item was |
---|
120 | available within that time. Otherwise (`block` is false), |
---|
121 | return an item if one is immediately available, else raise the |
---|
122 | ``Empty`` exception (`timeout` is ignored in that case). |
---|
123 | """ |
---|
124 | |
---|
125 | self.not_empty.acquire() |
---|
126 | try: |
---|
127 | if not block: |
---|
128 | if self._empty(): |
---|
129 | raise Empty |
---|
130 | elif timeout is None: |
---|
131 | while self._empty(): |
---|
132 | self.not_empty.wait() |
---|
133 | else: |
---|
134 | if timeout < 0: |
---|
135 | raise ValueError("'timeout' must be a positive number") |
---|
136 | endtime = _time() + timeout |
---|
137 | while self._empty(): |
---|
138 | remaining = endtime - _time() |
---|
139 | if remaining <= 0.0: |
---|
140 | raise Empty |
---|
141 | self.not_empty.wait(remaining) |
---|
142 | item = self._get() |
---|
143 | self.not_full.notify() |
---|
144 | return item |
---|
145 | finally: |
---|
146 | self.not_empty.release() |
---|
147 | |
---|
148 | def get_nowait(self): |
---|
149 | """Remove and return an item from the queue without blocking. |
---|
150 | |
---|
151 | Only get an item if one is immediately available. Otherwise |
---|
152 | raise the ``Empty`` exception. |
---|
153 | """ |
---|
154 | |
---|
155 | return self.get(False) |
---|
156 | |
---|
157 | # Override these methods to implement other queue organizations |
---|
158 | # (e.g. stack or priority queue). |
---|
159 | # These will only be called with appropriate locks held |
---|
160 | |
---|
161 | # Initialize the queue representation |
---|
162 | def _init(self, maxsize): |
---|
163 | self.maxsize = maxsize |
---|
164 | self.queue = deque() |
---|
165 | |
---|
166 | def _qsize(self): |
---|
167 | return len(self.queue) |
---|
168 | |
---|
169 | # Check whether the queue is empty |
---|
170 | def _empty(self): |
---|
171 | return not self.queue |
---|
172 | |
---|
173 | # Check whether the queue is full |
---|
174 | def _full(self): |
---|
175 | return self.maxsize > 0 and len(self.queue) == self.maxsize |
---|
176 | |
---|
177 | # Put a new item in the queue |
---|
178 | def _put(self, item): |
---|
179 | self.queue.append(item) |
---|
180 | |
---|
181 | # Get an item from the queue |
---|
182 | def _get(self): |
---|
183 | return self.queue.popleft() |
---|