cache.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933
  1. # -*- coding: utf-8 -*-
  2. """
  3. werkzeug.contrib.cache
  4. ~~~~~~~~~~~~~~~~~~~~~~
  5. The main problem with dynamic Web sites is, well, they're dynamic. Each
  6. time a user requests a page, the webserver executes a lot of code, queries
  7. the database, renders templates until the visitor gets the page he sees.
  8. This is a lot more expensive than just loading a file from the file system
  9. and sending it to the visitor.
  10. For most Web applications, this overhead isn't a big deal but once it
  11. becomes, you will be glad to have a cache system in place.
  12. How Caching Works
  13. =================
  14. Caching is pretty simple. Basically you have a cache object lurking around
  15. somewhere that is connected to a remote cache or the file system or
  16. something else. When the request comes in you check if the current page
  17. is already in the cache and if so, you're returning it from the cache.
  18. Otherwise you generate the page and put it into the cache. (Or a fragment
  19. of the page, you don't have to cache the full thing)
  20. Here is a simple example of how to cache a sidebar for 5 minutes::
  21. def get_sidebar(user):
  22. identifier = 'sidebar_for/user%d' % user.id
  23. value = cache.get(identifier)
  24. if value is not None:
  25. return value
  26. value = generate_sidebar_for(user=user)
  27. cache.set(identifier, value, timeout=60 * 5)
  28. return value
  29. Creating a Cache Object
  30. =======================
  31. To create a cache object you just import the cache system of your choice
  32. from the cache module and instantiate it. Then you can start working
  33. with that object:
  34. >>> from werkzeug.contrib.cache import SimpleCache
  35. >>> c = SimpleCache()
  36. >>> c.set("foo", "value")
  37. >>> c.get("foo")
  38. 'value'
  39. >>> c.get("missing") is None
  40. True
  41. Please keep in mind that you have to create the cache and put it somewhere
  42. you have access to it (either as a module global you can import or you just
  43. put it into your WSGI application).
  44. :copyright: 2007 Pallets
  45. :license: BSD-3-Clause
  46. """
  47. import errno
  48. import os
  49. import platform
  50. import re
  51. import tempfile
  52. import warnings
  53. from hashlib import md5
  54. from time import time
  55. from .._compat import integer_types
  56. from .._compat import iteritems
  57. from .._compat import string_types
  58. from .._compat import text_type
  59. from .._compat import to_native
  60. from ..posixemulation import rename
  61. try:
  62. import cPickle as pickle
  63. except ImportError: # pragma: no cover
  64. import pickle
  65. warnings.warn(
  66. "'werkzeug.contrib.cache' is deprecated as of version 0.15 and will"
  67. " be removed in version 1.0. It has moved to https://github.com"
  68. "/pallets/cachelib.",
  69. DeprecationWarning,
  70. stacklevel=2,
  71. )
  72. def _items(mappingorseq):
  73. """Wrapper for efficient iteration over mappings represented by dicts
  74. or sequences::
  75. >>> for k, v in _items((i, i*i) for i in xrange(5)):
  76. ... assert k*k == v
  77. >>> for k, v in _items(dict((i, i*i) for i in xrange(5))):
  78. ... assert k*k == v
  79. """
  80. if hasattr(mappingorseq, "items"):
  81. return iteritems(mappingorseq)
  82. return mappingorseq
  83. class BaseCache(object):
  84. """Baseclass for the cache systems. All the cache systems implement this
  85. API or a superset of it.
  86. :param default_timeout: the default timeout (in seconds) that is used if
  87. no timeout is specified on :meth:`set`. A timeout
  88. of 0 indicates that the cache never expires.
  89. """
  90. def __init__(self, default_timeout=300):
  91. self.default_timeout = default_timeout
  92. def _normalize_timeout(self, timeout):
  93. if timeout is None:
  94. timeout = self.default_timeout
  95. return timeout
  96. def get(self, key):
  97. """Look up key in the cache and return the value for it.
  98. :param key: the key to be looked up.
  99. :returns: The value if it exists and is readable, else ``None``.
  100. """
  101. return None
  102. def delete(self, key):
  103. """Delete `key` from the cache.
  104. :param key: the key to delete.
  105. :returns: Whether the key existed and has been deleted.
  106. :rtype: boolean
  107. """
  108. return True
  109. def get_many(self, *keys):
  110. """Returns a list of values for the given keys.
  111. For each key an item in the list is created::
  112. foo, bar = cache.get_many("foo", "bar")
  113. Has the same error handling as :meth:`get`.
  114. :param keys: The function accepts multiple keys as positional
  115. arguments.
  116. """
  117. return [self.get(k) for k in keys]
  118. def get_dict(self, *keys):
  119. """Like :meth:`get_many` but return a dict::
  120. d = cache.get_dict("foo", "bar")
  121. foo = d["foo"]
  122. bar = d["bar"]
  123. :param keys: The function accepts multiple keys as positional
  124. arguments.
  125. """
  126. return dict(zip(keys, self.get_many(*keys)))
  127. def set(self, key, value, timeout=None):
  128. """Add a new key/value to the cache (overwrites value, if key already
  129. exists in the cache).
  130. :param key: the key to set
  131. :param value: the value for the key
  132. :param timeout: the cache timeout for the key in seconds (if not
  133. specified, it uses the default timeout). A timeout of
  134. 0 idicates that the cache never expires.
  135. :returns: ``True`` if key has been updated, ``False`` for backend
  136. errors. Pickling errors, however, will raise a subclass of
  137. ``pickle.PickleError``.
  138. :rtype: boolean
  139. """
  140. return True
  141. def add(self, key, value, timeout=None):
  142. """Works like :meth:`set` but does not overwrite the values of already
  143. existing keys.
  144. :param key: the key to set
  145. :param value: the value for the key
  146. :param timeout: the cache timeout for the key in seconds (if not
  147. specified, it uses the default timeout). A timeout of
  148. 0 idicates that the cache never expires.
  149. :returns: Same as :meth:`set`, but also ``False`` for already
  150. existing keys.
  151. :rtype: boolean
  152. """
  153. return True
  154. def set_many(self, mapping, timeout=None):
  155. """Sets multiple keys and values from a mapping.
  156. :param mapping: a mapping with the keys/values to set.
  157. :param timeout: the cache timeout for the key in seconds (if not
  158. specified, it uses the default timeout). A timeout of
  159. 0 idicates that the cache never expires.
  160. :returns: Whether all given keys have been set.
  161. :rtype: boolean
  162. """
  163. rv = True
  164. for key, value in _items(mapping):
  165. if not self.set(key, value, timeout):
  166. rv = False
  167. return rv
  168. def delete_many(self, *keys):
  169. """Deletes multiple keys at once.
  170. :param keys: The function accepts multiple keys as positional
  171. arguments.
  172. :returns: Whether all given keys have been deleted.
  173. :rtype: boolean
  174. """
  175. return all(self.delete(key) for key in keys)
  176. def has(self, key):
  177. """Checks if a key exists in the cache without returning it. This is a
  178. cheap operation that bypasses loading the actual data on the backend.
  179. This method is optional and may not be implemented on all caches.
  180. :param key: the key to check
  181. """
  182. raise NotImplementedError(
  183. "%s doesn't have an efficient implementation of `has`. That "
  184. "means it is impossible to check whether a key exists without "
  185. "fully loading the key's data. Consider using `self.get` "
  186. "explicitly if you don't care about performance."
  187. )
  188. def clear(self):
  189. """Clears the cache. Keep in mind that not all caches support
  190. completely clearing the cache.
  191. :returns: Whether the cache has been cleared.
  192. :rtype: boolean
  193. """
  194. return True
  195. def inc(self, key, delta=1):
  196. """Increments the value of a key by `delta`. If the key does
  197. not yet exist it is initialized with `delta`.
  198. For supporting caches this is an atomic operation.
  199. :param key: the key to increment.
  200. :param delta: the delta to add.
  201. :returns: The new value or ``None`` for backend errors.
  202. """
  203. value = (self.get(key) or 0) + delta
  204. return value if self.set(key, value) else None
  205. def dec(self, key, delta=1):
  206. """Decrements the value of a key by `delta`. If the key does
  207. not yet exist it is initialized with `-delta`.
  208. For supporting caches this is an atomic operation.
  209. :param key: the key to increment.
  210. :param delta: the delta to subtract.
  211. :returns: The new value or `None` for backend errors.
  212. """
  213. value = (self.get(key) or 0) - delta
  214. return value if self.set(key, value) else None
  215. class NullCache(BaseCache):
  216. """A cache that doesn't cache. This can be useful for unit testing.
  217. :param default_timeout: a dummy parameter that is ignored but exists
  218. for API compatibility with other caches.
  219. """
  220. def has(self, key):
  221. return False
  222. class SimpleCache(BaseCache):
  223. """Simple memory cache for single process environments. This class exists
  224. mainly for the development server and is not 100% thread safe. It tries
  225. to use as many atomic operations as possible and no locks for simplicity
  226. but it could happen under heavy load that keys are added multiple times.
  227. :param threshold: the maximum number of items the cache stores before
  228. it starts deleting some.
  229. :param default_timeout: the default timeout that is used if no timeout is
  230. specified on :meth:`~BaseCache.set`. A timeout of
  231. 0 indicates that the cache never expires.
  232. """
  233. def __init__(self, threshold=500, default_timeout=300):
  234. BaseCache.__init__(self, default_timeout)
  235. self._cache = {}
  236. self.clear = self._cache.clear
  237. self._threshold = threshold
  238. def _prune(self):
  239. if len(self._cache) > self._threshold:
  240. now = time()
  241. toremove = []
  242. for idx, (key, (expires, _)) in enumerate(self._cache.items()):
  243. if (expires != 0 and expires <= now) or idx % 3 == 0:
  244. toremove.append(key)
  245. for key in toremove:
  246. self._cache.pop(key, None)
  247. def _normalize_timeout(self, timeout):
  248. timeout = BaseCache._normalize_timeout(self, timeout)
  249. if timeout > 0:
  250. timeout = time() + timeout
  251. return timeout
  252. def get(self, key):
  253. try:
  254. expires, value = self._cache[key]
  255. if expires == 0 or expires > time():
  256. return pickle.loads(value)
  257. except (KeyError, pickle.PickleError):
  258. return None
  259. def set(self, key, value, timeout=None):
  260. expires = self._normalize_timeout(timeout)
  261. self._prune()
  262. self._cache[key] = (expires, pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
  263. return True
  264. def add(self, key, value, timeout=None):
  265. expires = self._normalize_timeout(timeout)
  266. self._prune()
  267. item = (expires, pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
  268. if key in self._cache:
  269. return False
  270. self._cache.setdefault(key, item)
  271. return True
  272. def delete(self, key):
  273. return self._cache.pop(key, None) is not None
  274. def has(self, key):
  275. try:
  276. expires, value = self._cache[key]
  277. return expires == 0 or expires > time()
  278. except KeyError:
  279. return False
  280. _test_memcached_key = re.compile(r"[^\x00-\x21\xff]{1,250}$").match
  281. class MemcachedCache(BaseCache):
  282. """A cache that uses memcached as backend.
  283. The first argument can either be an object that resembles the API of a
  284. :class:`memcache.Client` or a tuple/list of server addresses. In the
  285. event that a tuple/list is passed, Werkzeug tries to import the best
  286. available memcache library.
  287. This cache looks into the following packages/modules to find bindings for
  288. memcached:
  289. - ``pylibmc``
  290. - ``google.appengine.api.memcached``
  291. - ``memcached``
  292. - ``libmc``
  293. Implementation notes: This cache backend works around some limitations in
  294. memcached to simplify the interface. For example unicode keys are encoded
  295. to utf-8 on the fly. Methods such as :meth:`~BaseCache.get_dict` return
  296. the keys in the same format as passed. Furthermore all get methods
  297. silently ignore key errors to not cause problems when untrusted user data
  298. is passed to the get methods which is often the case in web applications.
  299. :param servers: a list or tuple of server addresses or alternatively
  300. a :class:`memcache.Client` or a compatible client.
  301. :param default_timeout: the default timeout that is used if no timeout is
  302. specified on :meth:`~BaseCache.set`. A timeout of
  303. 0 indicates that the cache never expires.
  304. :param key_prefix: a prefix that is added before all keys. This makes it
  305. possible to use the same memcached server for different
  306. applications. Keep in mind that
  307. :meth:`~BaseCache.clear` will also clear keys with a
  308. different prefix.
  309. """
  310. def __init__(self, servers=None, default_timeout=300, key_prefix=None):
  311. BaseCache.__init__(self, default_timeout)
  312. if servers is None or isinstance(servers, (list, tuple)):
  313. if servers is None:
  314. servers = ["127.0.0.1:11211"]
  315. self._client = self.import_preferred_memcache_lib(servers)
  316. if self._client is None:
  317. raise RuntimeError("no memcache module found")
  318. else:
  319. # NOTE: servers is actually an already initialized memcache
  320. # client.
  321. self._client = servers
  322. self.key_prefix = to_native(key_prefix)
  323. def _normalize_key(self, key):
  324. key = to_native(key, "utf-8")
  325. if self.key_prefix:
  326. key = self.key_prefix + key
  327. return key
  328. def _normalize_timeout(self, timeout):
  329. timeout = BaseCache._normalize_timeout(self, timeout)
  330. if timeout > 0:
  331. timeout = int(time()) + timeout
  332. return timeout
  333. def get(self, key):
  334. key = self._normalize_key(key)
  335. # memcached doesn't support keys longer than that. Because often
  336. # checks for so long keys can occur because it's tested from user
  337. # submitted data etc we fail silently for getting.
  338. if _test_memcached_key(key):
  339. return self._client.get(key)
  340. def get_dict(self, *keys):
  341. key_mapping = {}
  342. have_encoded_keys = False
  343. for key in keys:
  344. encoded_key = self._normalize_key(key)
  345. if not isinstance(key, str):
  346. have_encoded_keys = True
  347. if _test_memcached_key(key):
  348. key_mapping[encoded_key] = key
  349. _keys = list(key_mapping)
  350. d = rv = self._client.get_multi(_keys)
  351. if have_encoded_keys or self.key_prefix:
  352. rv = {}
  353. for key, value in iteritems(d):
  354. rv[key_mapping[key]] = value
  355. if len(rv) < len(keys):
  356. for key in keys:
  357. if key not in rv:
  358. rv[key] = None
  359. return rv
  360. def add(self, key, value, timeout=None):
  361. key = self._normalize_key(key)
  362. timeout = self._normalize_timeout(timeout)
  363. return self._client.add(key, value, timeout)
  364. def set(self, key, value, timeout=None):
  365. key = self._normalize_key(key)
  366. timeout = self._normalize_timeout(timeout)
  367. return self._client.set(key, value, timeout)
  368. def get_many(self, *keys):
  369. d = self.get_dict(*keys)
  370. return [d[key] for key in keys]
  371. def set_many(self, mapping, timeout=None):
  372. new_mapping = {}
  373. for key, value in _items(mapping):
  374. key = self._normalize_key(key)
  375. new_mapping[key] = value
  376. timeout = self._normalize_timeout(timeout)
  377. failed_keys = self._client.set_multi(new_mapping, timeout)
  378. return not failed_keys
  379. def delete(self, key):
  380. key = self._normalize_key(key)
  381. if _test_memcached_key(key):
  382. return self._client.delete(key)
  383. def delete_many(self, *keys):
  384. new_keys = []
  385. for key in keys:
  386. key = self._normalize_key(key)
  387. if _test_memcached_key(key):
  388. new_keys.append(key)
  389. return self._client.delete_multi(new_keys)
  390. def has(self, key):
  391. key = self._normalize_key(key)
  392. if _test_memcached_key(key):
  393. return self._client.append(key, "")
  394. return False
  395. def clear(self):
  396. return self._client.flush_all()
  397. def inc(self, key, delta=1):
  398. key = self._normalize_key(key)
  399. return self._client.incr(key, delta)
  400. def dec(self, key, delta=1):
  401. key = self._normalize_key(key)
  402. return self._client.decr(key, delta)
  403. def import_preferred_memcache_lib(self, servers):
  404. """Returns an initialized memcache client. Used by the constructor."""
  405. try:
  406. import pylibmc
  407. except ImportError:
  408. pass
  409. else:
  410. return pylibmc.Client(servers)
  411. try:
  412. from google.appengine.api import memcache
  413. except ImportError:
  414. pass
  415. else:
  416. return memcache.Client()
  417. try:
  418. import memcache
  419. except ImportError:
  420. pass
  421. else:
  422. return memcache.Client(servers)
  423. try:
  424. import libmc
  425. except ImportError:
  426. pass
  427. else:
  428. return libmc.Client(servers)
  429. # backwards compatibility
  430. GAEMemcachedCache = MemcachedCache
  431. class RedisCache(BaseCache):
  432. """Uses the Redis key-value store as a cache backend.
  433. The first argument can be either a string denoting address of the Redis
  434. server or an object resembling an instance of a redis.Redis class.
  435. Note: Python Redis API already takes care of encoding unicode strings on
  436. the fly.
  437. .. versionadded:: 0.7
  438. .. versionadded:: 0.8
  439. `key_prefix` was added.
  440. .. versionchanged:: 0.8
  441. This cache backend now properly serializes objects.
  442. .. versionchanged:: 0.8.3
  443. This cache backend now supports password authentication.
  444. .. versionchanged:: 0.10
  445. ``**kwargs`` is now passed to the redis object.
  446. :param host: address of the Redis server or an object which API is
  447. compatible with the official Python Redis client (redis-py).
  448. :param port: port number on which Redis server listens for connections.
  449. :param password: password authentication for the Redis server.
  450. :param db: db (zero-based numeric index) on Redis Server to connect.
  451. :param default_timeout: the default timeout that is used if no timeout is
  452. specified on :meth:`~BaseCache.set`. A timeout of
  453. 0 indicates that the cache never expires.
  454. :param key_prefix: A prefix that should be added to all keys.
  455. Any additional keyword arguments will be passed to ``redis.Redis``.
  456. """
  457. def __init__(
  458. self,
  459. host="localhost",
  460. port=6379,
  461. password=None,
  462. db=0,
  463. default_timeout=300,
  464. key_prefix=None,
  465. **kwargs
  466. ):
  467. BaseCache.__init__(self, default_timeout)
  468. if host is None:
  469. raise ValueError("RedisCache host parameter may not be None")
  470. if isinstance(host, string_types):
  471. try:
  472. import redis
  473. except ImportError:
  474. raise RuntimeError("no redis module found")
  475. if kwargs.get("decode_responses", None):
  476. raise ValueError("decode_responses is not supported by RedisCache.")
  477. self._client = redis.Redis(
  478. host=host, port=port, password=password, db=db, **kwargs
  479. )
  480. else:
  481. self._client = host
  482. self.key_prefix = key_prefix or ""
  483. def _normalize_timeout(self, timeout):
  484. timeout = BaseCache._normalize_timeout(self, timeout)
  485. if timeout == 0:
  486. timeout = -1
  487. return timeout
  488. def dump_object(self, value):
  489. """Dumps an object into a string for redis. By default it serializes
  490. integers as regular string and pickle dumps everything else.
  491. """
  492. t = type(value)
  493. if t in integer_types:
  494. return str(value).encode("ascii")
  495. return b"!" + pickle.dumps(value)
  496. def load_object(self, value):
  497. """The reversal of :meth:`dump_object`. This might be called with
  498. None.
  499. """
  500. if value is None:
  501. return None
  502. if value.startswith(b"!"):
  503. try:
  504. return pickle.loads(value[1:])
  505. except pickle.PickleError:
  506. return None
  507. try:
  508. return int(value)
  509. except ValueError:
  510. # before 0.8 we did not have serialization. Still support that.
  511. return value
  512. def get(self, key):
  513. return self.load_object(self._client.get(self.key_prefix + key))
  514. def get_many(self, *keys):
  515. if self.key_prefix:
  516. keys = [self.key_prefix + key for key in keys]
  517. return [self.load_object(x) for x in self._client.mget(keys)]
  518. def set(self, key, value, timeout=None):
  519. timeout = self._normalize_timeout(timeout)
  520. dump = self.dump_object(value)
  521. if timeout == -1:
  522. result = self._client.set(name=self.key_prefix + key, value=dump)
  523. else:
  524. result = self._client.setex(
  525. name=self.key_prefix + key, value=dump, time=timeout
  526. )
  527. return result
  528. def add(self, key, value, timeout=None):
  529. timeout = self._normalize_timeout(timeout)
  530. dump = self.dump_object(value)
  531. return self._client.setnx(
  532. name=self.key_prefix + key, value=dump
  533. ) and self._client.expire(name=self.key_prefix + key, time=timeout)
  534. def set_many(self, mapping, timeout=None):
  535. timeout = self._normalize_timeout(timeout)
  536. # Use transaction=False to batch without calling redis MULTI
  537. # which is not supported by twemproxy
  538. pipe = self._client.pipeline(transaction=False)
  539. for key, value in _items(mapping):
  540. dump = self.dump_object(value)
  541. if timeout == -1:
  542. pipe.set(name=self.key_prefix + key, value=dump)
  543. else:
  544. pipe.setex(name=self.key_prefix + key, value=dump, time=timeout)
  545. return pipe.execute()
  546. def delete(self, key):
  547. return self._client.delete(self.key_prefix + key)
  548. def delete_many(self, *keys):
  549. if not keys:
  550. return
  551. if self.key_prefix:
  552. keys = [self.key_prefix + key for key in keys]
  553. return self._client.delete(*keys)
  554. def has(self, key):
  555. return self._client.exists(self.key_prefix + key)
  556. def clear(self):
  557. status = False
  558. if self.key_prefix:
  559. keys = self._client.keys(self.key_prefix + "*")
  560. if keys:
  561. status = self._client.delete(*keys)
  562. else:
  563. status = self._client.flushdb()
  564. return status
  565. def inc(self, key, delta=1):
  566. return self._client.incr(name=self.key_prefix + key, amount=delta)
  567. def dec(self, key, delta=1):
  568. return self._client.decr(name=self.key_prefix + key, amount=delta)
  569. class FileSystemCache(BaseCache):
  570. """A cache that stores the items on the file system. This cache depends
  571. on being the only user of the `cache_dir`. Make absolutely sure that
  572. nobody but this cache stores files there or otherwise the cache will
  573. randomly delete files therein.
  574. :param cache_dir: the directory where cache files are stored.
  575. :param threshold: the maximum number of items the cache stores before
  576. it starts deleting some. A threshold value of 0
  577. indicates no threshold.
  578. :param default_timeout: the default timeout that is used if no timeout is
  579. specified on :meth:`~BaseCache.set`. A timeout of
  580. 0 indicates that the cache never expires.
  581. :param mode: the file mode wanted for the cache files, default 0600
  582. """
  583. #: used for temporary files by the FileSystemCache
  584. _fs_transaction_suffix = ".__wz_cache"
  585. #: keep amount of files in a cache element
  586. _fs_count_file = "__wz_cache_count"
  587. def __init__(self, cache_dir, threshold=500, default_timeout=300, mode=0o600):
  588. BaseCache.__init__(self, default_timeout)
  589. self._path = cache_dir
  590. self._threshold = threshold
  591. self._mode = mode
  592. try:
  593. os.makedirs(self._path)
  594. except OSError as ex:
  595. if ex.errno != errno.EEXIST:
  596. raise
  597. self._update_count(value=len(self._list_dir()))
  598. @property
  599. def _file_count(self):
  600. return self.get(self._fs_count_file) or 0
  601. def _update_count(self, delta=None, value=None):
  602. # If we have no threshold, don't count files
  603. if self._threshold == 0:
  604. return
  605. if delta:
  606. new_count = self._file_count + delta
  607. else:
  608. new_count = value or 0
  609. self.set(self._fs_count_file, new_count, mgmt_element=True)
  610. def _normalize_timeout(self, timeout):
  611. timeout = BaseCache._normalize_timeout(self, timeout)
  612. if timeout != 0:
  613. timeout = time() + timeout
  614. return int(timeout)
  615. def _list_dir(self):
  616. """return a list of (fully qualified) cache filenames
  617. """
  618. mgmt_files = [
  619. self._get_filename(name).split("/")[-1] for name in (self._fs_count_file,)
  620. ]
  621. return [
  622. os.path.join(self._path, fn)
  623. for fn in os.listdir(self._path)
  624. if not fn.endswith(self._fs_transaction_suffix) and fn not in mgmt_files
  625. ]
  626. def _prune(self):
  627. if self._threshold == 0 or not self._file_count > self._threshold:
  628. return
  629. entries = self._list_dir()
  630. now = time()
  631. for idx, fname in enumerate(entries):
  632. try:
  633. remove = False
  634. with open(fname, "rb") as f:
  635. expires = pickle.load(f)
  636. remove = (expires != 0 and expires <= now) or idx % 3 == 0
  637. if remove:
  638. os.remove(fname)
  639. except (IOError, OSError):
  640. pass
  641. self._update_count(value=len(self._list_dir()))
  642. def clear(self):
  643. for fname in self._list_dir():
  644. try:
  645. os.remove(fname)
  646. except (IOError, OSError):
  647. self._update_count(value=len(self._list_dir()))
  648. return False
  649. self._update_count(value=0)
  650. return True
  651. def _get_filename(self, key):
  652. if isinstance(key, text_type):
  653. key = key.encode("utf-8") # XXX unicode review
  654. hash = md5(key).hexdigest()
  655. return os.path.join(self._path, hash)
  656. def get(self, key):
  657. filename = self._get_filename(key)
  658. try:
  659. with open(filename, "rb") as f:
  660. pickle_time = pickle.load(f)
  661. if pickle_time == 0 or pickle_time >= time():
  662. return pickle.load(f)
  663. else:
  664. os.remove(filename)
  665. return None
  666. except (IOError, OSError, pickle.PickleError):
  667. return None
  668. def add(self, key, value, timeout=None):
  669. filename = self._get_filename(key)
  670. if not os.path.exists(filename):
  671. return self.set(key, value, timeout)
  672. return False
  673. def set(self, key, value, timeout=None, mgmt_element=False):
  674. # Management elements have no timeout
  675. if mgmt_element:
  676. timeout = 0
  677. # Don't prune on management element update, to avoid loop
  678. else:
  679. self._prune()
  680. timeout = self._normalize_timeout(timeout)
  681. filename = self._get_filename(key)
  682. try:
  683. fd, tmp = tempfile.mkstemp(
  684. suffix=self._fs_transaction_suffix, dir=self._path
  685. )
  686. with os.fdopen(fd, "wb") as f:
  687. pickle.dump(timeout, f, 1)
  688. pickle.dump(value, f, pickle.HIGHEST_PROTOCOL)
  689. rename(tmp, filename)
  690. os.chmod(filename, self._mode)
  691. except (IOError, OSError):
  692. return False
  693. else:
  694. # Management elements should not count towards threshold
  695. if not mgmt_element:
  696. self._update_count(delta=1)
  697. return True
  698. def delete(self, key, mgmt_element=False):
  699. try:
  700. os.remove(self._get_filename(key))
  701. except (IOError, OSError):
  702. return False
  703. else:
  704. # Management elements should not count towards threshold
  705. if not mgmt_element:
  706. self._update_count(delta=-1)
  707. return True
  708. def has(self, key):
  709. filename = self._get_filename(key)
  710. try:
  711. with open(filename, "rb") as f:
  712. pickle_time = pickle.load(f)
  713. if pickle_time == 0 or pickle_time >= time():
  714. return True
  715. else:
  716. os.remove(filename)
  717. return False
  718. except (IOError, OSError, pickle.PickleError):
  719. return False
  720. class UWSGICache(BaseCache):
  721. """Implements the cache using uWSGI's caching framework.
  722. .. note::
  723. This class cannot be used when running under PyPy, because the uWSGI
  724. API implementation for PyPy is lacking the needed functionality.
  725. :param default_timeout: The default timeout in seconds.
  726. :param cache: The name of the caching instance to connect to, for
  727. example: mycache@localhost:3031, defaults to an empty string, which
  728. means uWSGI will cache in the local instance. If the cache is in the
  729. same instance as the werkzeug app, you only have to provide the name of
  730. the cache.
  731. """
  732. def __init__(self, default_timeout=300, cache=""):
  733. BaseCache.__init__(self, default_timeout)
  734. if platform.python_implementation() == "PyPy":
  735. raise RuntimeError(
  736. "uWSGI caching does not work under PyPy, see "
  737. "the docs for more details."
  738. )
  739. try:
  740. import uwsgi
  741. self._uwsgi = uwsgi
  742. except ImportError:
  743. raise RuntimeError(
  744. "uWSGI could not be imported, are you running under uWSGI?"
  745. )
  746. self.cache = cache
  747. def get(self, key):
  748. rv = self._uwsgi.cache_get(key, self.cache)
  749. if rv is None:
  750. return
  751. return pickle.loads(rv)
  752. def delete(self, key):
  753. return self._uwsgi.cache_del(key, self.cache)
  754. def set(self, key, value, timeout=None):
  755. return self._uwsgi.cache_update(
  756. key, pickle.dumps(value), self._normalize_timeout(timeout), self.cache
  757. )
  758. def add(self, key, value, timeout=None):
  759. return self._uwsgi.cache_set(
  760. key, pickle.dumps(value), self._normalize_timeout(timeout), self.cache
  761. )
  762. def clear(self):
  763. return self._uwsgi.cache_clear(self.cache)
  764. def has(self, key):
  765. return self._uwsgi.cache_exists(key, self.cache) is not None