diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d0903ea --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +docker-quobyte-plugin diff --git a/README.md b/README.md index cf0ff5b..4ade322 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,87 @@ -Quobyte volume plug-in for Docker -================================= +# Quobyte volume plug-in for Docker -Setup: -* create a user in Quobyte for the plug-in: +Tested with `CentOS 7.2` and `Docker 1.10.3`. This plugin allows you to use [Quobyte](www.quobyte.com) with Docker without installing the Quobyte client on the host system (e.q. Rancher/CoreOS). + +## Build + +Get the code + +``` +$ go get -u github.com/quobyte/api +$ go get -u github.com/quobyte/go-quobyte-docker ``` - qmgmt -u user config add docker + +### Linux + ``` +$ go build -o docker-quobyte-plugin . +$ cp quobyte-docker-plugin /usr/libexec/docker/docker-quobyte-plugin +``` + +### OSX/MacOS -* set mandatory configuration in environment ``` -export QUOBYTE_API_USER=docker -export QUOBYTE_API_PASSWORD=... -export QUOBYTE_API_URL=http://:7860/ +$ GOOS=linux GOARCH=amd64 go build -o docker-quobyte-plugin +$ cp quobyte-docker-plugin /usr/libexec/docker/docker-quobyte-plugin +``` + +## Setup + +### Create a user in Quobyte for the plug-in: + +This step is optional. + +``` +$ qmgmt -u user config add docker +``` + +### Set mandatory configuration in environment + +``` +$ export QUOBYTE_API_USER=docker +$ export QUOBYTE_API_PASSWORD=... +$ export QUOBYTE_API_URL=http://:7860/ # host[:port][,host:port] or SRV record name -export QUOBYTE_REGISTRY=quobyte.corp +$ export QUOBYTE_REGISTRY=quobyte.corp +``` + +### Install systemd files Set the variables in systemd/docker-quobyte.env.sample + +``` +$ cp systemd/docker-quobyte.env.sample /etc/quobyte/docker-quobyte.env +$ cp docker-quobyte-plugin /usr/libexec/docker/ +$ cp systemd/* /lib/systemd/system + +$ systemctl daemon-reload +$ systemctl start docker-quobyte-plugin +$ systemctl enable docker-quobyte-plugin +$ systemctl status docker-quobyte-plugin ``` -* Start the plug-in as root (with above environment) -``` -quobyte-docker-volume.py +## Examples + +### Create a volume + +``` +$ docker volume create --driver quobyte --name +# Set user and group of the volume +$ docker volume create --driver quobyte --name --opt user=docker --opt group=docker ``` -Examples: +### Delete a volume ``` -# docker volume create --driver quobyte --name --opt volume_config=MyConfig -# docker volume create --driver quobyte --name -# docker volume rm -# docker run --volume-driver=quobyte -v :path +$ docker volume rm ``` -* Install systemd files -Set the variables in systemd/docker-quobyte.env.sample +### List all volumes ``` -cp systemd/docker-quobyte.env.sample /etc/quobyte/docker-quobyte.env -cp quobyte-docker-volume.py /usr/libexec/docker/ -cp systemd/* /lib/systemd/system +$ docker volume ls +``` + +### Attach volume to container -systemctl enable docker-quobyte-plugin -systemctl status docker-quobyte-plugin +``` +$ docker run --volume-driver=quobyte -v :/vol busybox sh -c 'echo "Hello World" > /vol/hello.txt' ``` diff --git a/main.go b/main.go new file mode 100644 index 0000000..a31cd9c --- /dev/null +++ b/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "log" + "os" + + "github.com/docker/go-plugins-helpers/volume" +) + +const quobyteID string = "quobyte" + +// Mandatory configuration +var qmgmtUser string +var qmgmtPassword string +var quobyteAPIURL string +var quobyteRegistry string + +// Optional configuration +var mountQuobytePath string +var mountQuobyteOptions string + +func main() { + readMandatoryConfig() + readOptionalConfig() + + if err := os.MkdirAll(mountQuobytePath, 0555); err != nil { + log.Println(err.Error()) + } + + if !isMounted(mountQuobytePath) { + log.Printf("Mounting Quobyte namespace in %s", mountQuobytePath) + mountAll() + } + + qDriver := newQuobyteDriver(quobyteAPIURL, qmgmtUser, qmgmtPassword) + handler := volume.NewHandler(qDriver) + log.Println(handler.ServeUnix("root", quobyteID)) +} diff --git a/quobyte-docker-volume.py b/quobyte-docker-volume.py deleted file mode 100755 index 847483d..0000000 --- a/quobyte-docker-volume.py +++ /dev/null @@ -1,249 +0,0 @@ -#!/usr/bin/python2.7 -# -# A Docker volume plug-in -# -# Copyright 2016 Quobyte Inc. All rights reserved. -# -# Usage: -# - set mandatory configuration (see below) -# - Run as root -# -# Examples: -# docker create --driver quobyte --name --opt volume_config=MyConfig -# docker create --driver quobyte --name -# docker rm -# docker run --volume-driver=quobyte -v :path - -from BaseHTTPServer import BaseHTTPRequestHandler -import BaseHTTPServer -import json -import os -import os.path -import socket -import time - - -def getenv_mandatory(name): - result = os.getenv(name) - if not result: - raise BaseException("Please set " + name + " in environment") - return result - -# Mandatory configuration -QMGMT_USER = getenv_mandatory("QUOBYTE_API_USER") -QMGMT_PASSWORD = getenv_mandatory("QUOBYTE_API_PASSWORD") -QUOBYTE_API_URL = getenv_mandatory("QUOBYTE_API_URL") -# host[:port][,host:port] or SRV record name -QUOBYTE_REGISTRY = getenv_mandatory("QUOBYTE_REGISTRY") - -# Optional configuration -MOUNT_QUOBYTE_PATH = "" -MOUNT_QUOBYTE_OPTIONS = "-o user_xattr" -QMGMT_PATH = "" -DEFAULT_VOLUME_CONFIGURATION = "BASE" - - -# Constants -PLUGIN_DIRECTORY = '/run/docker/plugins/' -PLUGIN_SOCKET = PLUGIN_DIRECTORY + 'quobyte.sock' -MOUNT_DIRECTORY = '/run/docker/quobyte/mnt' - - -def read_optional_config(): - global MOUNT_QUOBYTE_PATH - MOUNT_QUOBYTE_PATH = os.getenv("MOUNT_QUOBYTE_PATH") - global QMGMT_PATH - QMGMT_PATH = os.getenv("QMGMT_PATH") - options = os.getenv("MOUNT_QUOBYTE_OPTIONS") - if options: - global MOUNT_QUOBYTE_OPTIONS - MOUNT_QUOBYTE_OPTIONS = options - config = os.getenv("DEFAULT_VOLUME_CONFIGURATION") - if config: - global DEFAULT_VOLUME_CONFIGURATION - DEFAULT_VOLUME_CONFIGURATION = config - - -def mount_all(): - binary = "mount.quobyte" - if MOUNT_QUOBYTE_PATH: - binary = os.path.join(MOUNT_QUOBYTE_PATH, binary) - mnt_cmd = (binary + " " + MOUNT_QUOBYTE_OPTIONS + " " + - QUOBYTE_REGISTRY + "/ " + MOUNT_DIRECTORY) - print mnt_cmd - return os.system(mnt_cmd) - - -def qmgmt(params): - binary = "qmgmt" - if not QUOBYTE_API_URL: - print "Please configure API URL" - raise Exception() - if QMGMT_PATH: - binary = os.path.join(QMGMT_PATH, binary) - cmdline = binary + " -u " + QUOBYTE_API_URL + " " + params - print cmdline - exitcode = os.system(cmdline) - print "==", exitcode - return exitcode == 0 - - -def volume_create(name, volume_config): - return qmgmt( - "volume create " + - name + - " root root " + - volume_config + - " 777") - - -def volume_delete(name): - return qmgmt("volume delete -f " + name) - - -def volume_exists(name): - return qmgmt("volume resolve " + name) - - -def is_mounted(mountpath): - mounts = open('/proc/mounts') - for mount in mounts: - if mount.split()[1] == mountpath: - return True - return False - - -class UDSServer(BaseHTTPServer.HTTPServer): - address_family = socket.AF_UNIX - socket_type = socket.SOCK_STREAM - - def __init__(self, server_address, RequestHandlerClass): - try: - os.unlink(server_address) - except OSError: - if os.path.exists(server_address): - raise - self.socket = socket.socket(self.address_family, self.socket_type) - BaseHTTPServer.HTTPServer.__init__( - self, server_address, RequestHandlerClass) - - def server_bind(self): - self.socket.bind(self.server_address) - - def server_activate(self): - self.socket.listen(1) - - def server_close(self): - self.socket.close() - - def fileno(self): - return self.socket.fileno() - - def close_request(self, request): - request.close() - - def get_request(self): - return self.socket.accept()[0], 'uds' - - -class DockerHandler(BaseHTTPRequestHandler): - mount_paths = {} - - def get_request(self): - length = int(self.headers['content-length']) - return json.loads(self.rfile.read(length)) - - def respond(self, msg): - self.send_response(200) - self.send_header( - "Content-type", - "application/vnd.docker.plugins.v1+json") - self.end_headers() - print "Responding with", json.dumps(msg) - self.wfile.write(json.dumps(msg)) - - def do_POST(self): - print self.get_request() - if self.path == "/Plugin.Activate": - self.respond({"Implements": ["VolumeDriver"]}) - elif self.path == "/VolumeDriver.Create": - self.volume_driver_create() - elif self.path == "/VolumeDriver.Remove": - self.volume_driver_remove() - elif self.path == "/VolumeDriver.Path" or self.path == "/VolumeDriver.Mount": - self.volume_driver_mount() - elif self.path == "/VolumeDriver.Get": - self.volume_driver_get() - elif self.path == "/VolumeDriver.Unmount": - self.respond({"Err": ""}) - elif self.path == "/VolumeDriver.List": - self.volume_driver_list() - else: - print "Unknown API operation:", self.path - self.respond({"Err": "Unknown API operation: " + self.path}) - - def volume_driver_create(self): - volume_config = DEFAULT_VOLUME_CONFIGURATION - request = self.get_request() - name = request["Name"] - if 'Opts' in request and request[ - 'Opts'] and 'volume_configuration' in request['Opts']: - volume_config = request['Opts']['volume_configuration'] - - volume_create(name, volume_config) - mountpoint = os.path.join(MOUNT_DIRECTORY, name) - while not os.path.exists(mountpoint): - print "Waiting for", mountpoint - time.sleep(1) - self.respond({"Err": ""}) - - def volume_driver_remove(self): - name = self.get_request()["Name"] - if not volume_exists(name): - self.respond({"Err": ""}) - return - if volume_delete(name): - self.respond({"Err": ""}) - else: - self.respond({"Err": "Could not delete " + name}) - - def volume_driver_mount(self): - name = self.get_request()["Name"] - mountpoint = os.path.join(MOUNT_DIRECTORY, name) - if os.path.exists(mountpoint): - self.respond({"Err": "", "Mountpoint": mountpoint}) - else: - self.respond({"Err": "Not mounted: " + name}) - - def volume_driver_get(self): - name = self.get_request()["Name"] - mountpoint = os.path.join(MOUNT_DIRECTORY, name) - if os.path.exists(mountpoint): - self.respond({"Volume": {"Name": name, "Mountpoint": mountpoint}, "Err": ""}) - else: - self.respond({"Err": "Not mounted: " + name}) - - def volume_driver_list(self): - volumes = os.listdir(MOUNT_DIRECTORY) - result = [{"Name": v, "Mountpoint": os.path.join( - MOUNT_DIRECTORY, v)} for v in volumes] - self.respond({"Volumes": result, "Err": ""}) - - -if __name__ == '__main__': - read_optional_config() - try: - os.makedirs(MOUNT_DIRECTORY) - except OSError as error: - if error.errno != 17: - raise error - try: - os.makedirs(PLUGIN_DIRECTORY) - except OSError as error: - if error.errno != 17: - raise error - if not is_mounted(MOUNT_DIRECTORY): - print "Mounting Quobyte namespace in", MOUNT_DIRECTORY - mount_all() - print 'Starting server, use to stop' - UDSServer(PLUGIN_SOCKET, DockerHandler).serve_forever() diff --git a/quobyte_driver.go b/quobyte_driver.go new file mode 100644 index 0000000..15f1d35 --- /dev/null +++ b/quobyte_driver.go @@ -0,0 +1,117 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "sync" + + "github.com/docker/go-plugins-helpers/volume" + quobyte_api "github.com/quobyte/api" +) + +type quobyteDriver struct { + client *quobyte_api.QuobyteClient + m *sync.Mutex +} + +func newQuobyteDriver(apiURL string, username string, password string) quobyteDriver { + driver := quobyteDriver{ + client: quobyte_api.NewQuobyteClient(apiURL, username, password), + m: &sync.Mutex{}, + } + + return driver +} + +func (driver quobyteDriver) Create(request volume.Request) volume.Response { + log.Printf("Creating volume %s\n", request.Name) + driver.m.Lock() + defer driver.m.Unlock() + + user, group := "root", "root" + + if usr, ok := request.Options["user"]; ok { + user = usr + } + + if grp, ok := request.Options["group"]; ok { + group = grp + } + + if _, err := driver.client.CreateVolume(request.Name, user, group); err != nil { + return volume.Response{Err: err.Error()} + } + + return volume.Response{Err: ""} +} + +func (driver quobyteDriver) Remove(request volume.Request) volume.Response { + log.Printf("Removing volume %s\n", request.Name) + driver.m.Lock() + defer driver.m.Unlock() + + if err := driver.client.DeleteVolumeByName(request.Name); err != nil { + return volume.Response{Err: err.Error()} + } + + return volume.Response{Err: ""} +} + +func (driver quobyteDriver) Mount(request volume.Request) volume.Response { + driver.m.Lock() + defer driver.m.Unlock() + mPoint := filepath.Join(mountQuobytePath, request.Name) + log.Printf("Mounting volume %s on %s\n", request.Name, mPoint) + if fi, err := os.Lstat(mPoint); err != nil || !fi.IsDir() { + return volume.Response{Err: fmt.Sprintf("%v not mounted", mPoint)} + } + + return volume.Response{Err: "", Mountpoint: mPoint} +} + +func (driver quobyteDriver) Path(request volume.Request) volume.Response { + return volume.Response{Mountpoint: filepath.Join(mountQuobytePath, request.Name)} +} + +func (driver quobyteDriver) Unmount(request volume.Request) volume.Response { + return volume.Response{} +} + +func (driver quobyteDriver) Get(request volume.Request) volume.Response { + driver.m.Lock() + defer driver.m.Unlock() + + mPoint := filepath.Join(mountQuobytePath, request.Name) + + if fi, err := os.Lstat(mPoint); err != nil || !fi.IsDir() { + return volume.Response{Err: fmt.Sprintf("%v not mounted", mPoint)} + } + + return volume.Response{Volume: &volume.Volume{Name: request.Name, Mountpoint: mPoint}} +} + +func (driver quobyteDriver) List(request volume.Request) volume.Response { + driver.m.Lock() + defer driver.m.Unlock() + + var vols []*volume.Volume + files, err := ioutil.ReadDir(mountQuobytePath) + if err != nil { + return volume.Response{Err: err.Error()} + } + + for _, entry := range files { + if entry.IsDir() { + vols = append(vols, &volume.Volume{Name: entry.Name(), Mountpoint: filepath.Join(mountQuobytePath, entry.Name())}) + } + } + + return volume.Response{Volumes: vols} +} + +func (driver quobyteDriver) Capabilities(request volume.Request) volume.Response { + return volume.Response{Capabilities: volume.Capability{Scope: "local"}} +} diff --git a/systemd/docker-quobyte-plugin.service b/systemd/docker-quobyte-plugin.service index 6abc105..11822f6 100644 --- a/systemd/docker-quobyte-plugin.service +++ b/systemd/docker-quobyte-plugin.service @@ -1,13 +1,13 @@ [Unit] Description=Docker Quobyte Plugin -Documentation=https://github.com/quobyte/docker-volume +Documentation=https://github.com/johscheuer/go-quobyte-docker Before=docker.service After=network.target docker-quobyte-plugin.socket Requires=docker-quobyte-plugin.socket docker.service [Service] EnvironmentFile=/etc/quobyte/docker-quobyte.env -ExecStart=/usr/bin/python /usr/libexec/docker/quobyte-docker-volume.py +ExecStart=/usr/libexec/docker/docker-quobyte-plugin [Install] WantedBy=multi-user.target diff --git a/systemd/docker-quobyte-plugin.socket b/systemd/docker-quobyte-plugin.socket index 97f9f81..7beec42 100644 --- a/systemd/docker-quobyte-plugin.socket +++ b/systemd/docker-quobyte-plugin.socket @@ -3,7 +3,7 @@ Description=Docker Quobyte plugin Socket for the API Documentation=https://github.com/quobyte/docker-volume [Socket] -ListenStream=/run/docker/plugins/quobyte.sock +ListenStream=/var/run/docker/plugins/quobyte.sock [Install] WantedBy=sockets.target diff --git a/util.go b/util.go new file mode 100644 index 0000000..e0dcb9a --- /dev/null +++ b/util.go @@ -0,0 +1,64 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "strings" +) + +func readOptionalConfig() { + mountQuobytePath = os.Getenv("MOUNT_QUOBYTE_PATH") + if len(mountQuobyteOptions) == 0 { + mountQuobytePath = "/run/docker/quobyte/mnt" + } + mountQuobyteOptions = os.Getenv("MOUNT_QUOBYTE_OPTIONS") + if len(mountQuobyteOptions) == 0 { + mountQuobyteOptions = "-o user_xattr" + } +} + +func readMandatoryConfig() { + qmgmtUser = getMandatoryEnv("QUOBYTE_API_USER") + qmgmtPassword = getMandatoryEnv("QUOBYTE_API_PASSWORD") + quobyteAPIURL = getMandatoryEnv("QUOBYTE_API_URL") + quobyteRegistry = getMandatoryEnv("QUOBYTE_REGISTRY") +} + +func getMandatoryEnv(name string) string { + env := os.Getenv(name) + if len(env) < 0 { + log.Fatalf("Please set %s in environment\n", name) + } + + return env +} + +func isMounted(mountPath string) bool { + content, err := ioutil.ReadFile("/proc/mounts") + if err != nil { + log.Println(err) + } + for _, mount := range strings.Split(string(content), "\n") { + splitted := strings.Split(mount, " ") + if len(splitted) < 2 { + continue + } + + if splitted[1] == mountPath { + log.Printf("Found Mountpoint: %s\n", mountPath) + return true + } + } + + return false +} + +func mountAll() { + cmdStr := fmt.Sprintf("mount %s -t quobyte %s %s", mountQuobyteOptions, fmt.Sprintf("%s/", quobyteRegistry), mountQuobytePath) + if out, err := exec.Command("/bin/sh", "-c", cmdStr).CombinedOutput(); err != nil { + log.Fatalln(string(out)) + } +}