i'd also appreciate an option for docker install latest ala
https://docs.docker.com/installation/binaries/


On Mon, Aug 18, 2014 at 12:29 PM, Cory Johns <email address hidden>
wrote:

> Cory Johns has proposed merging lp:~johnsca/charm-helpers/docker into
> lp:charm-helpers.
>
> Requested reviews:
>   Charm Helper Maintainers (charm-helpers)
>
> For more details, see:
> https://code.launchpad.net/~johnsca/charm-helpers/docker/+merge/231226
>
> Added helpers for managing Docker containers in a charm, using the
> services framework.  Includes documentation.
> --
> https://code.launchpad.net/~johnsca/charm-helpers/docker/+merge/231226
> Your team Charm Helper Maintainers is requested to review the proposed
> merge of lp:~johnsca/charm-helpers/docker into lp:charm-helpers.
>
> === added directory 'charmhelpers/contrib/docker'
> === added file 'charmhelpers/contrib/docker/__init__.py'
> --- charmhelpers/contrib/docker/__init__.py     1970-01-01 00:00:00 +0000
> +++ charmhelpers/contrib/docker/__init__.py     2014-08-18 16:28:55 +0000
> @@ -0,0 +1,287 @@
> +
> +import os
> +import subprocess
> +
> +from charmhelpers import fetch
> +from charmhelpers.core import host
> +from charmhelpers.core import hookenv
> +from charmhelpers.core.services.base import ManagerCallback
> +from charmhelpers.core.services.helpers import RelationContext
> +
> +
> +def install_docker():
> +    """
> +    Install the Docker tools.
> +    """
> +    fetch.apt_install(['docker.io'])
> +    if os.path.exists('/usr/local/bin/docker'):
> +        os.unlink('/usr/local/bin/docker')
> +    os.symlink('/usr/bin/docker.io', '/usr/local/bin/docker')
> +    with open('/etc/bash_completion.d/docker.io', 'a') as fp:
> +        fp.write('\ncomplete -F _docker docker')
> +
> +
> +def install_docker_unstable():
> +    """
> +    Install the Docker tools from the unstable (but potentially more
> +    up-to-date) repository.
> +    """
> +    fetch.add_source('deb https://get.docker.io/ubuntu docker main',
> +                     key='36A1D7869245C8950F966E92D8576A8BA88D21E9')
> +    fetch.apt_update(fatal=True)
> +    fetch.apt_install(['lxc-docker'])
> +
> +
> +def docker_pull(container_name, registry=None, username=None,
> password=None):
> +    """
> +    Fetch a container from the Docker registry.
> +
> +    If :param:`registry` is given, the container will be pulled from that
> +    registry instead of the default.  If the registry requires
> authentication,
> +    provide the :param:`username` and :param:`password` arguments.
> +    """
> +    if registry is not None:
> +        container_name = '/'.join([registry, container_name])
> +        if any(username, password):
> +            subprocess.check_call([
> +                'docker', 'login', '-u', username, '-p', password])
> +    subprocess.check_call(['docker', 'pull', container_name])
> +
> +
> +class DockerCallback(ManagerCallback):
> +    """
> +    ServiceManager callback to manage starting up a Docker container.
> +
> +    Can be referenced as ``docker_start`` or ``docker_stop``, and performs
> +    the appropriate action.  Requires one or more of
> +    :class:`DockerPortMappings`, :class:`DockerVolumes`,
> +    :class:`DockerContainerArgs`, and :class:`DockerRelation` to be
> +    included in the ``required_data`` section of the services definition.
> +
> +    Example::
> +
> +      manager = services.ServiceManager([{
> +          'service': 'dockerfile/rethinkdb',
> +          'required_data': [
> +              DockerPortMappings({
> +                  80: 8080,
> +                  28015: 28015,
> +                  29015: 29015,
> +              }),
> +              DockerVolumes(mapped_volumes={'data': '/rethinkdb'}),
> +              DockerContainerArgs(
> +                  '--bind', 'all',
> +                  '--canonical-address',
> hookenv.unit_get('public-address'),
> +                  '--canonical-address',
> hookenv.unit_get('private-address'),
> +                  '--machine-name', socket.gethostname().replace('-',
> '_'),
> +              ),
> +          ],
> +          'start': docker_start,
> +          'stop': docker_stop,
> +      }])
> +    """
> +    def __call__(self, manager, service_name, event_name):
> +        container_id_file = os.path.join(hookenv.charm_dir(),
> 'CONTAINER_ID')
> +        if os.path.exists(container_id_file):
> +            container_id = host.read_file(container_id_file)
> +            subprocess.check_call(['docker', 'stop', container_id])
> +            os.remove(container_id_file)
> +        if event_name == 'start':
> +            subprocess.check_call(
> +                ['docker', 'run', '-d', '--cidfile', container_id_file] +
> +                self.get_volume_args(manager, service_name) +
> +                self.get_port_args(manager, service_name) +
> +                [service_name] +
> +                self.get_container_args(manager, service_name))
> +
> +    def _get_args(self, manager, service_name, arg_type):
> +        args = []
> +        service = manager.get_service(service_name)
> +        for provider in service['required_data']:
> +            if isinstance(provider, arg_type):
> +                args.extend(provider.build_args())
> +        return args
> +
> +    def get_port_args(self, manager, service_name):
> +        return self._get_args(manager, service_name, DockerPortMappings)
> +
> +    def get_container_args(self, manager, service_name):
> +        return self._get_args(manager, service_name, DockerContainerArgs)
> +
> +    def get_volume_args(self, manager, service_name):
> +        return self._get_args(manager, service_name, DockerVolumes)
> +
> +
> +class DockerPortMappings(dict):
> +    """
> +    Context class for use in ``required_data`` representing a mapping of
> +    ports from the host to the container.
> +
> +    Example::
> +
> +      manager = services.ServiceManager([{
> +          'required_data': [
> +              DockerPortMappings({
> +                  80: 8080,
> +                  28015: 28015,
> +                  29015: 29015,
> +              }),
> +          ],
> +      }])
> +
> +    """
> +    def build_args(self):
> +        ports = []
> +        for src, dst in self.iteritems():
> +            ports.extend(['-p', '{}:{}'.format(src, dst)])
> +        return ports
> +
> +
> +class DockerVolumes(object):
> +    """
> +    Context class for use in ``required_data`` representing Docker
> volumes.
> +
> +    Example::
> +
> +      manager = services.ServiceManager([{
> +          'required_data': [
> +              DockerVolumes(mapped_volumes={'data': '/rethinkdb'}),
> +          ],
> +          'start': docker_start,
> +          'stop': docker_stop,
> +      }])
> +    """
> +    def __init__(self, volumes=None, named_volumes=None,
> mapped_volumes=None):
> +        """
> +        :param volumes: List of mutable data volumes to create.
> +        :type volumes: list
> +        :param named_volumes: Mapping of names to mutable data volumes to
> create.
> +        :type volumes: dict
> +        :param mapped_volumes: Mapping of host paths to container paths.
> +        :type volumes: dict
> +        """
> +        assert any([volumes, named_volumes, mapped_volumes]),\
> +            'Must provide at least one of: volumes, named_volumes,
> mapped_volumes'
> +        self.volumes = volumes or []
> +        self.named_volumes = named_volumes or {}
> +        self.mapped_volumes = mapped_volumes or {}
> +
> +    def build_args(self):
> +        args = []
> +        for volume in self.volumes:
> +            args.extend(['-v', volume])
> +        for name, volume in self.named_volumes.iteritems():
> +            args.extend(['-v', volume, '--name', name])
> +        for host_path, volume in self.mapped_volumes.iteritems():
> +            if not os.path.isabs(host_path):
> +                host_path = os.path.join(hookenv.charm_dir(), host_path)
> +            host.mkdir(host_path)
> +            args.extend(['-v', ':'.join([host_path, volume])])
> +        return args
> +
> +
> +class DockerContainerArgs(object):
> +    """
> +    Context class for use in ``required_data`` representing arguments to
> be
> +    passed to the Docker container.
> +
> +    Example::
> +
> +      manager = services.ServiceManager([{
> +          'required_data': [
> +              DockerContainerArgs(
> +                  '--bind', 'all',
> +                  '--canonical-address',
> hookenv.unit_get('public-address'),
> +                  '--canonical-address',
> hookenv.unit_get('private-address'),
> +                  '--machine-name', socket.gethostname().replace('-',
> '_'),
> +              ),
> +          ],
> +          'start': docker_start,
> +          'stop': docker_stop,
> +      }])
> +    """
> +    def __init__(self, *args, **kwargs):
> +        """
> +        Can accept either a list of arg strings, or a kwarg mapping of arg
> +        names to values.  If kwargs are given, the names are prepended
> +        with '--' and have any underscores converted to dashes.
> +
> +        For example, `DockerContainerArgs(my_opt='foo')` becomes
> `['--my-opt', 'foo']`.
> +
> +        If you need to run a specific command other than the container
> default, it
> +        should be the first argument.
> +        """
> +        self.args = list(args)
> +        for key, value in kwargs.iteritems():
> +            self.args.extend(['--'+key.replace('_', '-'), value])
> +
> +    def build_args(self):
> +        return self.args
> +
> +
> +class DockerRelation(RelationContext, DockerContainerArgs):
> +    """
> +    :class:`~charmhelpers.core.services.helpers.RelationContext` subclass
> +    for use as a base class for contexts representing a relation to
> another
> +    Docker container, which could be another service, or a peer within the
> +    same service.
> +
> +    Example::
> +
> +      class DatabaseContainerRelation(DockerRelation):
> +          name = 'db'
> +          interface = 'mysql-docker'
> +          required_keys = ['db-host', 'db-name']
> +
> +      manager = services.ServiceManager([{
> +          'required_data': [DatabaseContainerRelation()],
> +          'start': docker_start,
> +          'stop': docker_stop,
> +      }])
> +    """
> +    name = None
> +    interface = None
> +    required_keys = []
> +
> +    def map(self, relation_settings):
> +        """
> +        Translate the relation settings from a single unit into a list of
> +        arguments to be passed to the Docker container.  This may be
> called
> +        multiple times, once for each unit, and the resulting arguments
> will
> +        all be passed to the container.
> +
> +        The default implementation simply appends ``--`` to the relation
> setting
> +        name, so that ``{'private-address': '10.0.0.1'}`` is transformed
> +        to ``['--private-address', '10.0.0.1']``.
> +
> +        For example, the following subclass would translate the
> ``private-address``
> +        value from each peer to a ``--join`` argument for each connected
> peer::
> +
> +          class ClusterPeers(DockerRelation):
> +              name = 'cluster'
> +              interface = 'cluster'
> +              required_keys = ['private-address']
> +              port = 29015
> +
> +              def map(self, relation_settings):
> +                  return [
> +                      '--join', '{}:{}'.format(
> +                          relation_settings['private-address'],
> +                          self.port
> +                      )
> +                  ]
> +        """
> +        args = []
> +        for key, value in relation_settings.iteritems():
> +            args.extend(['--{}'.format(key), str(value)])
> +        return args
> +
> +    def build_args(self):
> +        args = []
> +        for unit in self.get(self.name, []):
> +            args.extend(self.map(unit))
> +        return args
> +
> +
> +# Convenience aliases for Docker
> +docker_start = docker_stop = DockerCallback()
>
> === modified file 'charmhelpers/core/host.py'
> --- charmhelpers/core/host.py   2014-08-05 20:52:59 +0000
> +++ charmhelpers/core/host.py   2014-08-18 16:28:55 +0000
> @@ -157,6 +157,11 @@
>          target.write(content)
>
>
> +def read_file(path):
> +    with open(path) as fp:
> +        return fp.read()
> +
> +
>  def fstab_remove(mp):
>      """Remove the given mountpoint entry from /etc/fstab
>      """
>
> === modified file 'charmhelpers/core/services/base.py'
> --- charmhelpers/core/services/base.py  2014-08-05 21:28:01 +0000
> +++ charmhelpers/core/services/base.py  2014-08-18 16:28:55 +0000
> @@ -31,6 +31,7 @@
>              {
>                  "service": <service name>,
>                  "required_data": <list of required data contexts>,
> +                "provided_data": <list of provided data contexts>,
>                  "data_ready": <one or more callbacks>,
>                  "data_lost": <one or more callbacks>,
>                  "start": <one or more callbacks>,
> @@ -44,6 +45,10 @@
>          of 'data_ready' and 'start' callbacks executed.  See `is_ready()`
> for more
>          information.
>
> +        The 'provided_data' list should contain relation data providers,
> most likely
> +        a subclass of
> :class:`charmhelpers.core.services.helpers.RelationContext`,
> +        that will indicate a set of data to set on a given relation.
> +
>          The 'data_ready' value should be either a single callback, or a
> list of
>          callbacks, to be called when all items in 'required_data' pass
> `is_ready()`.
>          Each callback will be called with the service name as the only
> parameter.
> @@ -123,12 +128,20 @@
>              self.reconfigure_services()
>
>      def provide_data(self):
> +        """
> +        Set the relation data for each provider in the ``provided_data``
> list.
> +
> +        A provider must have a `name` attribute, which indicates which
> relation
> +        to set data on, and a `provide_data()` method, which returns a
> dict of
> +        data to set.
> +        """
>          hook_name = hookenv.hook_name()
>          for service in self.services.values():
>              for provider in service.get('provided_data', []):
>                  if re.match(r'{}-relation-(joined|changed)'.format(
> provider.name), hook_name):
>                      data = provider.provide_data()
> -                    if provider._is_ready(data):
> +                    _ready = provider._is_ready(data) if
> hasattr(provider, '_is_ready') else data
> +                    if _ready:
>                          hookenv.relation_set(None, data)
>
>      def reconfigure_services(self, *service_names):
>
> === added file 'docs/api/charmhelpers.contrib.docker.rst'
> --- docs/api/charmhelpers.contrib.docker.rst    1970-01-01 00:00:00 +0000
> +++ docs/api/charmhelpers.contrib.docker.rst    2014-08-18 16:28:55 +0000
> @@ -0,0 +1,8 @@
> +charmhelpers.contrib.docker package
> +===================================
> +
> +.. automodule:: charmhelpers.contrib.docker
> +    :members:
> +    :undoc-members:
> +    :show-inheritance:
> +
>
> === modified file 'docs/api/charmhelpers.contrib.rst'
> --- docs/api/charmhelpers.contrib.rst   2014-08-05 21:28:01 +0000
> +++ docs/api/charmhelpers.contrib.rst   2014-08-18 16:28:55 +0000
> @@ -6,6 +6,7 @@
>      charmhelpers.contrib.ansible
>      charmhelpers.contrib.charmhelpers
>      charmhelpers.contrib.charmsupport
> +    charmhelpers.contrib.docker
>      charmhelpers.contrib.hahelpers
>      charmhelpers.contrib.network
>      charmhelpers.contrib.openstack
>
> === added file 'docs/examples/docker.rst'
> --- docs/examples/docker.rst    1970-01-01 00:00:00 +0000
> +++ docs/examples/docker.rst    2014-08-18 16:28:55 +0000
> @@ -0,0 +1,119 @@
> +Managing Docker with the Services Framework
> +===========================================
> +
> +Docker containers can be easily managed with
> +the :mod:`services framework <charmhelpers.core.services.base>` and the
> +helpers provided in :mod:`charmhelpers.contrib.docker`.  Using these
> +helpers and classes will manage pulling the container, blocking on
> +neccessary data (such as from required config values, or other services),
> +starting the container with the appropriate arguments, and restarting the
> +container whenever any relevant configuration data has changed.  And the
> +resulting charm will be very declarative and thus should be easy to
> follow.
> +
> +
> +Installing Docker and a Container
> +---------------------------------
> +
> +Your install hook will probably just consist of the following lines::
> +
> +  # hooks/install
> +
> +  from charmhelpers.contrib.docker import install_docker, docker_pull
> +
> +  install_docker()
> +  docker_pull('my_container')
> +
> +The :func:`~charmhelpers.contrib.docker.docker_pull` helper also has
> +support for pulling from a custom registry, with authentication.
> +
> +
> +Defining Your Docker Container Configuration
> +--------------------------------------------
> +
> +The common configuration for a Docker container includes port mappings,
> +volumes and / or volume mappings, and arguments for the container itself
> +which may be built from various sources such as config options, unit data,
> +or relations to other services.  By using the various classes and
> +the :class:`docker_start() <charmhelpers.contrib.docker.DockerCallback>`
> +and :class:`docker_stop() <charmhelpers.contrib.docker.DockerCallback>`
> +helpers from :mod:`charmhelpers.contrib.docker` with the
> +:class:`~charmhelpers.core.services.base.ServiceManager`, you can declare
> +this configuration in a single location, and then just call
> +:meth:`manager.manage()
> <charmhelpers.core.services.base.ServiceManager.manage>`
> +to handle your hooks.
> +
> +An example, from the `RethinkDB Docker Charm <
> https://github.com/bcsaller/juju-docker/>`_,
> +which uses most of the options available, is below::
> +
> +  # hooks/hooks.py
> +  # (symlinked to from start, stop, config-changed,
> intracluster-relation-joined, etc)
> +
> +  import socket
> +  from charmhelpers.core import hookenv
> +  from charmhelpers.core import services
> +  from charmhelpers.contrib import docker
> +
> +  class ClusterPeers(docker.DockerRelation):
> +      name = 'intracluster'
> +      interface = 'rethinkdb-cluster'
> +      port = 29015
> +
> +      def map(self, relation_settings):
> +          return [
> +              '--join', '{}:{}'.format(
> +                  relation_settings['private-address'],
> +                  self.port
> +              )
> +          ]
> +
> +
> +  class WebsiteRelation(services.helpers.RelationContext):
> +      name = 'website'
> +      interface = 'http'
> +
> +      def provide_data(self):
> +          return {'hostname': hookenv.unit_private_ip(), 'port': 80}
> +
> +
> +  config = hookenv.config()
> +  manager = services.ServiceManager([{
> +      'service': 'dockerfile/rethinkdb',
> +      'ports': [80, 28015, 29015],
> +      'provided_data': [WebsiteRelation()],
> +      'required_data': [
> +          config if config.get('storage-path') else {},
> +          DockerPortMappings({
> +              80: 8080,
> +              28015: 28015,
> +              29015: 29015,
> +          }),
> +          DockerVolumes(mapped_volumes={config.get('storage-path'):
> '/rethinkdb'}),
> +          DockerContainerArgs(
> +              bind='all',
> +              canonical_address=hookenv.unit_get('public-address'),
> +              canonical_address=hookenv.unit_get('private-address'),
> +              machine_name=socket.gethostname().replace('-', '_'),
> +          ),
> +          ClusterPeers(),
> +      ],
> +      'start': docker_start,
> +      'stop': docker_stop,
> +  }])
> +  manager.manage()
> +
> +In this example, once the ``storage-path`` config option has been set,
> +the container will be started with something like the following
> command-line::
> +
> +  docker run -d --cidfile CONTAINER_ID -v <storage-path>:/rethinkdb -p
> 80:8080 \
> +    -p 28015:28015 -p 29015:29015 dockerfile/rethinkdb --bind all \
> +    --canoncal-address <public-address> --canonical-address
> <private-address> \
> +    --machine-name <machine-name> --join <peer-addr>:<peer-port>
> +
> +Additionally, whenever a new peer is added via ``juju add-relation``, the
> container
> +will be automatically restarted with the additional ``--join`` argument.
> +
> +
> +Additional Steps
> +----------------
> +
> +Actually, that's it.  =)
>
> === added directory 'tests/contrib/docker'
> === added file 'tests/contrib/docker/__init__.py'
> === added file 'tests/contrib/docker/test_docker.py'
> --- tests/contrib/docker/test_docker.py 1970-01-01 00:00:00 +0000
> +++ tests/contrib/docker/test_docker.py 2014-08-18 16:28:55 +0000
> @@ -0,0 +1,118 @@
> +import mock
> +import unittest
> +
> +from charmhelpers.contrib import docker
> +
> +
> +class TestDockerHelpers(unittest.TestCase):
> +    def test_port_mappings(self):
> +        mappings = docker.DockerPortMappings({
> +            '80': '8080',
> +            '5000': '5000',
> +        })
> +        self.assertIn(mappings.build_args(), [
> +            ['-p', '80:8080', '-p', '5000:5000'],
> +            ['-p', '5000:5000', '-p', '80:8080'],
> +        ])
> +
> +    def test_container_args(self):
> +        args = docker.DockerContainerArgs(
> +            'command',
> +            'arg',
> +            named_arg='value',
> +        )
> +        self.assertEqual(args.build_args(), ['command', 'arg',
> '--named-arg', 'value'])
> +
> +    @mock.patch('charmhelpers.core.host.mkdir')
> +    @mock.patch('charmhelpers.core.hookenv.charm_dir')
> +    def test_volumes(self, charm_dir, mkdir):
> +        charm_dir.return_value = 'charm_dir'
> +        vols = docker.DockerVolumes(
> +            volumes=['vol1'],
> +            named_volumes={'vol2_name': 'vol2'},
> +            mapped_volumes={'host1': 'vol3'},
> +        )
> +        self.assertEqual(vols.build_args(), [
> +            '-v', 'vol1',
> +            '-v', 'vol2', '--name', 'vol2_name',
> +            '-v', 'charm_dir/host1:vol3'])
> +        mkdir.assert_called_once_with('charm_dir/host1')
> +        mkdir.reset_mock()
> +        vols = docker.DockerVolumes(mapped_volumes={'/host1': 'vol4'})
> +        self.assertEqual(vols.build_args(), ['-v', '/host1:vol4'])
> +        mkdir.assert_called_once_with('/host1')
> +
> +    @mock.patch('charmhelpers.core.host.mkdir')
> +    @mock.patch('os.path.exists')
> +    @mock.patch('os.remove')
> +    @mock.patch('subprocess.check_call')
> +    @mock.patch('charmhelpers.core.hookenv.charm_dir')
> +    @mock.patch('charmhelpers.core.host.read_file')
> +    def test_docker_callback(self, read_file, charm_dir, check_call,
> exists, remove, mkdir):
> +        charm_dir.return_value = 'charm_dir'
> +        read_file.return_value = 'cid'
> +        exists.return_value = True
> +        manager = mock.Mock()
> +        manager.get_service.return_value = {
> +            'service': 'dockeruser/container',
> +            'required_data': [
> +                docker.DockerPortMappings({'80': '8080'}),
> +                docker.DockerVolumes(mapped_volumes={'data': '/path'}),
> +                docker.DockerContainerArgs(bind='all'),
> +            ],
> +        }
> +        callback = docker.DockerCallback()
> +        callback(manager, 'dockeruser/container', 'start')
> +        exists.assert_called_once_with('charm_dir/CONTAINER_ID')
> +        read_file.assert_called_once_with('charm_dir/CONTAINER_ID')
> +        check_call.assert_has_call(['docker', 'stop', 'cid'])
> +        remove.assert_called_once_with('charm_dir/CONTAINER_ID')
> +        check_call.assert_has_call([
> +            'docker', 'run', '-d', '-cidfile', 'charm_dir/CONTAINER_ID',
> +            '-v', 'charm_dir/data', '/path',
> +            '-p', '80:8080',
> +            'dockeruser/container',
> +            '--bind', 'all',
> +        ])
> +
> +
> @mock.patch('charmhelpers.core.services.helpers.RelationContext.get_data')
> +    def test_docker_relation(self, get_data):
> +        relation = docker.DockerRelation()
> +        relation.name = 'name'
> +        relation.required_keys = ['key1']
> +        relation.update({
> +            'name': [
> +                {'key1': 'val1'},
> +                {'key2': 'val2'},
> +            ],
> +        })
> +        self.assertEqual(set(relation.build_args()), set([
> +            '--key1', 'val1', '--key2', 'val2',
> +        ]))
> +
> +    @mock.patch('charmhelpers.contrib.docker.open', create=True)
> +    @mock.patch('os.symlink')
> +    @mock.patch('os.unlink')
> +    @mock.patch('os.path.exists')
> +    @mock.patch('charmhelpers.fetch.apt_install')
> +    def test_install_docker(self, apt_install, exists, unlink, symlink,
> mopen):
> +        exists.return_value = True
> +        fp = mopen.return_value.__enter__.return_value
> +        docker.install_docker()
> +        assert apt_install.called
> +        assert exists.called
> +        assert unlink.called
> +        assert symlink.called
> +        assert fp.write.called
> +
> +    @mock.patch('charmhelpers.contrib.docker.fetch')
> +    def test_install_docker_unstable(self, fetch):
> +        docker.install_docker_unstable()
> +        assert fetch.add_source.called
> +        assert fetch.apt_update.called
> +        assert fetch.apt_install.called
> +
> +    @mock.patch('subprocess.check_call')
> +    def test_docker_pull(self, check_call):
> +        docker.docker_pull('foo')
> +        check_call.assert_called_once_with(['docker', 'pull', 'foo'])
>
>
>