Source code for bob.db.base.driver

#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Mon 13 Aug 2012 16:19:18 CEST

"""This module defines, among other less important constructions, a management
interface that can be used by Bob to display information about the database and
manage installed files.

import os
import abc
import six

def dbshell(arguments):
  """Drops you into a database shell"""

  if len(arguments.files) != 1:
    raise RuntimeError(
        "Something is wrong this database is supposed to be of type SQLite, but you have more than one data file available: %s" % argument.files)

  if arguments.type == 'sqlite':
    prog = 'sqlite3'
    raise RuntimeError("Error auxiliary database file '%s' cannot be used to initiate a database shell connection (type='%s')" % (
        dbfile, arguments.type))

  cmdline = [prog, arguments.files[0]]

  import subprocess

    if arguments.dryrun:
      print("[dry-run] exec '%s'" % ' '.join(cmdline))
      return 0
      p = subprocess.Popen(cmdline)
  except OSError as e:
    # occurs when the file is not executable or not found
    print("Error executing '%s': %s (%d)" % (' '.join(cmdline), e.strerror,
    import sys

  except KeyboardInterrupt:  # the user CTRL-C'ed
    import signal
    os.kill(, signal.SIGTERM)
    return signal.SIGTERM

  return p.returncode

def dbshell_command(subparsers):
  """Adds a new dbshell subcommand to your subparser"""

  parser = subparsers.add_parser('dbshell', help=dbshell.__doc__)
  parser.add_argument("-n", "--dry-run", dest="dryrun", default=False,
                      help="does not actually run, just prints what would do instead")

def upload(arguments):
  """Uploads generated metadata to the Idiap build server"""

  import pkg_resources
  basedir = pkg_resources.resource_filename('bob.db.%s' %, '')
  assert basedir, "Database and package names do not match. Your declared " \
      "database name should be <name>, if your package is called bob.db.<name>"

  # check all files exist
  for p in arguments.files:
    if not os.path.exists(p):
      raise IOError("Metadata file `%s' is not available. Did you run "
                    "`create' before attempting to upload?" % (p,))

  # compress
  import tarfile
  import tempfile
  import six.moves.urllib
  import six.moves.http_client
  import shutil

  parsed_url = six.moves.urllib.parse.urlparse(arguments.url)

  with tempfile.TemporaryFile() as tmpfile:

    # if you get here, all files are there, ready to package
    print("Compressing metadata files to temporary file...")

    f =, mode='w:bz2')
    for k, p in enumerate(arguments.files):
      n = os.path.relpath(p, basedir)
      print("+ [%d/%d] %s" % (k + 1, len(arguments.files), n))
      f.add(p, n)

    # print what we are going to do
    target_path = '/'.join((parsed_url.path, + ".tar.bz2"))
    print("Uploading protocol files to %s" % target_path)

    if parsed_url.scheme in ('', 'file'): #local file upload
        shutil.copyfileobj(tmpfile, open(target_path, 'wb'))
      except (shutil.Error, IOError) as e:
        # maybe no file location? try next steps
        print("Error: %s" % e)

    # if you get to this point, it is because it is a network transfer
    if parsed_url.scheme == 'https':
      dav_server = six.moves.http_client.HTTPSConnection(parsed_url.netloc)
      dav_server = six.moves.http_client.HTTPConnection(parsed_url.netloc)

    # copy tmpfile to DAV server
    import base64
    import getpass
    from six.moves import input
    print("Authorization requested by server %s" % parsed_url.netloc)
    username = input('Username: ')
    username = username.encode('ascii')
    password = getpass.getpass(prompt='Password: ')
    password = password.encode('ascii')
    upass = base64.encodestring(b'%s:%s' % \
        (username, password)).decode('ascii')[:-1]
    headers = {'Authorization': 'Basic %s' % upass}
    dav_server.request('PUT', target_path, tmpfile, headers=headers)
    res = dav_server.getresponse()
    response =

    if not (200 <= res.status < 300):
	    raise IOError(response)
      print("Uploaded %s (status: %d)" % (target_path, res.status))

def upload_command(subparsers):
  """Adds a new 'upload' subcommand to your parser"""

  # default destination for the file to be uploaded is on the local directory
  curdir = 'file://' + os.path.realpath(os.curdir)

  parser = subparsers.add_parser('upload', help=upload.__doc__)
  parser.add_argument("url", default=curdir, nargs='?', help='Pass the URL ' \
      'for uploading your contribution (if not set, uses default: ' \

  return parser

def download(arguments):
  """Downloads and uncompresses meta data generated files from Idiap


    arguments (argparse.Namespace): A set of arguments passed by the
      command-line parser


    int: A POSIX compliant return value of ``0`` if the download is successful,
    or ``1`` in case it is not.


    IOError: if metafiles exist and ``--force`` was not passed

    urllib2.HTTPError: if the target resource does not exist on the webserver


  # What should happen as a combination of flags. Legend:
  # 0 - Exit, with status 0
  # X - Download, overwrite if there
  # R - Raise exception, err
  # +----------+-----------+----------+--------+
  # | complete | --missing | --force  |  none  |
  # +----------+-----------+----------+--------+
  # |   yes    |     0     |    X     |   R    |
  # +----------+-----------+----------+--------+
  # |   no     |     X     |    X     |   X    |
  # +----------+-----------+----------+--------+

  if not arguments.files:
    print("Skipping download of metadata files for bob.db.%s: no files "
          "declared" %

  # Check we're complete in terms of metafiles
  complete = True
  for p in arguments.files:
    if not os.path.exists(p):
      complete = False

  if complete:
    if arguments.missing:
      print("Skipping download of metadata files for `bob.db.%s': complete" %
      return 0
    elif arguments.force:
      print("Re-downloading metafiles for `bob.db.%s'" %
      raise IOError("Metadata files are already available. Remove metadata "
                    "files before attempting download or --force")

  # if you get here, all files aren't there, unpack
  source_url = os.path.join(arguments.source, + ".tar.bz2")

  target_dir = arguments.test_dir  # test case

  if not target_dir:  # puts files on the root of the installed package

    import pkg_resources
      target_dir = pkg_resources.resource_filename('bob.db.%s' %
                                         , '')
    except ImportError as e:
      raise ImportError("The package `bob.db.%s' is not currently "
                        "installed. N.B.: The database and package names **must** "
                        "match. Your package should be named `bob.db.%s', if the driver "
                        "name for your database is `%s'. Check." % (3 * (,)))

  # download file from Idiap server, unpack and remove it
  import sys
  import tempfile
  import tarfile
  import pkg_resources
  from .utils import safe_tarmembers
  if sys.version_info[0] <= 2:
    import urllib2 as urllib
    import urllib.request as urllib

  print ("Extracting url `%s' into `%s'" % (source_url, target_dir))
  u = urllib.urlopen(source_url)
  f = tempfile.NamedTemporaryFile(suffix=".tar.bz2")
  open(, 'wb').write(
  t =, mode='r:bz2')
  members = list(safe_tarmembers(t))
  for k, m in enumerate(members):
    print("x [%d/%d] %s" % (k + 1, len(members),,))
    t.extract(m, target_dir)

def download_command(subparsers):
  """Adds a new 'download' subcommand to your parser"""

  from argparse import SUPPRESS

  if 'DOCSERVER' in os.environ:
    USE_SERVER = os.environ['DOCSERVER']
    USE_SERVER = ''

  parser = subparsers.add_parser('download', help=download.__doc__)
                      default="%s/software/bob/databases/latest/" % USE_SERVER)
  group = parser.add_mutually_exclusive_group(required=False)
  group.add_argument("--force", action='store_true',
                     default=False, help="Overwrite existing database files?")
  group.add_argument("--missing", action='store_true',
                     default=False, help="Only downloads if files are missing")
  parser.add_argument("--test-dir", help=SUPPRESS)

  return parser

def print_files(arguments):
  """Prints the current location of raw database files."""

  print ("Files for database '%s':" %
  for k in arguments.files:

  return 0

def files_command(subparsers):
  """Adds a new 'files' subcommand to your parser"""

  parser = subparsers.add_parser('files', help=print_files.__doc__)

  return parser

def version(arguments):
  """Outputs the database version"""

  print('%s == %s' % (, arguments.version))

  return 0

def version_command(subparsers):

  parser = subparsers.add_parser('version', help=version.__doc__)

  return parser

[docs]@six.add_metaclass(abc.ABCMeta) class Interface(object): """Base manager for Bob databases You should derive and implement an Interface object on every ``bob.db`` package you create. """
[docs] @abc.abstractmethod def name(self): '''The name of this database Returns: str: a Python-conforming name for this database. This **must** match the package name. If the package is named ````, then this function must return ``foo``. ''' return
[docs] @abc.abstractmethod def files(self): '''List of meta-data files for the package to be downloaded/uploaded This function should normally return an empty list, except in case the database being implemented requires download/upload of metadata files that are **not** kept in its (git) repository. Returns: list: A python iterable with all metadata files needed. The paths listed by this method should correspond to full paths (not relative ones) w.r.t. the database package implementing it. This is normally achieved by using ``pkg_resources.resource_filename()``. ''' return
[docs] @abc.abstractmethod def version(self): '''The version of this package Returns: str: The current version number defined in ```` ''' return
[docs] @abc.abstractmethod def type(self): '''The type of auxiliary files you have for this database Returns: str: A string defining the type of database implemented. You can return only two values on this function, either ``sqlite`` or ``text``. If you return ``sqlite``, then we append special actions such as ``dbshell`` on ``bob_dbmanage`` automatically for you. Otherwise, we don't. ''' return
[docs] def setup_parser(self, parser, short_description, long_description): '''Sets up the base parser for this database. Parameters: short_description (str): A short description (one-liner) for this database long_description (str): A more involved explanation of this database Returns: argparse.ArgumentParser: a subparser, ready so you can add commands on ''' from argparse import RawDescriptionHelpFormatter # creates a top-level parser for this database top_level = parser.add_parser(, formatter_class=RawDescriptionHelpFormatter, help=short_description, description=long_description) type = self.type() files = self.files() top_level.set_defaults( top_level.set_defaults(version=self.version()) top_level.set_defaults(type=type) top_level.set_defaults(files=files) subparsers = top_level.add_subparsers(title="subcommands") # adds some stock commands version_command(subparsers) if files: upload_command(subparsers) download_command(subparsers) if type in ('sqlite',): dbshell_command(subparsers) if files is not None: files_command(subparsers) return subparsers
[docs] @abc.abstractmethod def add_commands(self, parser): '''Adds commands to a given :py:class:`argparse.ArgumentParser` This method, effectively, allows you to define special commands that your database will be able to perform when called from the common driver like for example ``create`` or ``checkfiles``. You are not obliged to overwrite this method. If you do, you will have the chance to establish your own commands. You don't have to worry about stock commands such as :py:meth:`files` or :py:meth:`version`. They will be automatically hooked-in depending on the values you return for :py:meth:`type` and :py:meth:`files`. Parameters: parser (argparse.ArgumentParser): An instance of a parser that you can customize, i.e., call :py:meth:`argparse.ArgumentParser.add_argument` on. ''' return
__all__ = ('Interface',)