Source code for relstorage.cache.persistence

##############################################################################
#
# Copyright (c) 2009 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""
Helpers for various disk-based persistent storage format.

Doesn't actually do any persistence itself.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import errno
import logging
import os
import os.path
import sqlite3


from relstorage.adapters.sqlite.drivers import Sqlite3Driver
from relstorage.adapters.sqlite.dialect import SQ3_SUPPORTS_CTE

# Because we use a CTE in our default queries. In principle,
# we could probably re-write queries and run on 3.7.11; 3.7.0 was the
# oldest to support WAL mode, and 3.7.11 added multiple VALUE syntax.
# Prior to 3.7.6, you can't use LENGTH() on a BLOB column.
SQ3_IS_MIN_VERSION = SQ3_SUPPORTS_CTE

logger = log = logging.getLogger(__name__)

def _normalize_path(options):
    path = os.path.expanduser(os.path.expandvars(options.cache_local_dir))
    path = os.path.abspath(path)
    return path

def trace_file(options, prefix):
    # Return an open file for tracing to, if that is set up.
    # Otherwise, return nothing.

    # We choose a trace file based on ZEO_CACHE_TRACE. If it is
    # set to 'single', then we use a single file (not suitable for multiple
    # process, but records client opens/closes). If it is set to any other value,
    # we include a pid. If it is not set, we do nothing.
    trace = os.environ.get("ZEO_CACHE_TRACE")
    if not trace or not options.cache_local_dir:
        return None

    if trace == 'single':
        pid = 0
    else: # pragma: no cover
        pid = os.getpid()

    name = 'relstorage-trace-' + prefix + '.' + str(pid) + '.trace'

    parent_dir = _normalize_path(options)
    try:
        os.makedirs(parent_dir)
    except os.error:
        pass
    fname = os.path.join(parent_dir, name)
    try:
        tf = open(fname, 'ab') # pylint:disable=consider-using-with
    except IOError as e: # pragma: no cover
        log.warning("Cannot write tracefile %r (%s)", fname, e)
        tf = None
    else:
        log.info("opened tracefile %r", fname)
    return tf



[docs] def sqlite_files(options, prefix): """ Calculate the sqlite filename and return it, plus a function that will destroy the sqlite file. """ parent_dir = getattr(options, 'cache_local_dir', options) # Allow for memory and temporary databases (empty string): if parent_dir != ':memory:' and parent_dir: parent_dir = _normalize_path(options) try: # make it if needed. try to avoid a time-of-use/check # race (not that it matters here) os.makedirs(parent_dir) except os.error: pass fname = os.path.join(parent_dir, 'relstorage-cache2-' + prefix + '.sqlite3') wal_fname = fname + '-wal' shm_fname = fname + '-shm' def real_destroy(): logger.info("Replacing any existing cache at %s", fname) __quiet_remove(fname) __quiet_remove(wal_fname) __quiet_remove(shm_fname) destroy = real_destroy else: fname = parent_dir wal_fname = None def noop_destroy(): "Nothing to do." destroy = noop_destroy return fname, destroy
[docs] class Sqlite3TooOldError(ValueError): """Raised if the sqlite3 module is too old.""" def __init__(self, *_args): super().__init__( "Unable to use sqlite; minimum version is %s but this version is %s" % ( "3.8.3", sqlite3.sqlite_version_info ))
class _Driver(Sqlite3Driver): STATIC_AVAILABLE = SQ3_SUPPORTS_CTE DriverNotAvailableError = Sqlite3TooOldError CORRUPT_DB_EXCEPTIONS = (sqlite3.DatabaseError,) FAILURE_TO_OPEN_DB_EXCEPTIONS = (sqlite3.OperationalError, Sqlite3TooOldError)
[docs] def sqlite_connect(options, prefix, overwrite=False, **connect_kwargs): """ Return a DB-API Connection object. If the database is corrupted, the file will be removed and recreated. .. caution:: Using the connection as a context manager does **not** result in the connection being closed, only committed or rolled back. """ if not SQ3_IS_MIN_VERSION: # pragma: no cover raise Sqlite3TooOldError() fname, destroy = sqlite_files(options, prefix) corrupt_db_ex = CORRUPT_DB_EXCEPTIONS if overwrite: destroy() corrupt_db_ex = () connect_args = (fname,) try: connection = _Driver().connect_to_file( *connect_args, **connect_kwargs ) except corrupt_db_ex as e: __traceback_info__ = e, fname, destroy logger.exception("Corrupt cache database at %s; replacing", fname) destroy() connection = _Driver().connect_to_file( *connect_args, **connect_kwargs ) return connection
def __quiet_remove(path): try: os.unlink(path) return True except os.error as e: # TODO: Use FileNotFoundError on Python 3? log_meth = log.exception if e.errno == errno.ENOENT: log_meth = log.debug log_meth("Failed to remove %r", path) return False