Let's see how docker-machine can be adapted for Linux on z Systems. Caution: long post.
We will use the generic driver which uses an existing Linux image as starting point, and uses its ssh access to make it a Docker host. First step -- let's setup the build environment for machine.
machine is written in go, so we need to use an environment as described back in Portability Series: A Go Environment. I have just used the compiler in a container to build this.
go get github.com/docker/machineThen, add a file s390x-suse.go to /go/src/github.com/docker/machine/libmachine/provision/ , containing:
package provisionAdd s390x-redhat.go to the same directory:
import (
"bytes"
"fmt"
"os"
"strings"
"text/template"
"github.com/docker/machine/libmachine/auth"
"github.com/docker/machine/libmachine/drivers"
"github.com/docker/machine/libmachine/engine"
"github.com/docker/machine/libmachine/log"
"github.com/docker/machine/libmachine/mcnutils"
"github.com/docker/machine/libmachine/provision/pkgaction"
"github.com/docker/machine/libmachine/provision/serviceaction"
"github.com/docker/machine/libmachine/ssh"
"github.com/docker/machine/libmachine/swarm"
)
var (
s3suseEngineConfigTemplate = `[Service]
ExecStart=/usr/bin/docker daemon -H tcp://0.0.0.0:{{.DockerPort}} -H unix:///var/run/docker.sock --storage-driver {{.EngineOptions.StorageDriver}} --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}} {{ range .EngineOptions.Labels }}--label {{.}} {{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} {{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} {{ end }}
MountFlags=slave
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
Environment={{range .EngineOptions.Env}}{{ printf "%q" . }} {{end}}
`
)
func init() {
Register("SUSE-s390x", &RegisteredProvisioner{
New: News390xSUSEProvisioner,
})
}
func News390xSUSEProvisioner(d drivers.Driver) Provisioner {
return &s390xSUSEProvisioner{
GenericProvisioner: GenericProvisioner{
DockerOptionsDir: "/etc/docker",
DaemonOptionsFile: "/etc/systemd/system/docker.service",
OsReleaseId: "sles",
Packages: []string{
"curl", "sudo",
},
Driver: d,
},
}
}
type s390xSUSEProvisioner struct {
GenericProvisioner
}
func (provisioner *s390xSUSEProvisioner) SSHCommand(args string) (string, error) {
client, err := drivers.GetSSHClientFromDriver(provisioner.Driver)
if err != nil {
return "", err
}
// redhat needs "-t" for tty allocation on ssh therefore we check for the
// external client and add as needed.
// Note: CentOS 7.0 needs multiple "-tt" to force tty allocation when ssh has
// no local tty.
switch c := client.(type) {
case ssh.ExternalClient:
c.BaseArgs = append(c.BaseArgs, "-tt")
client = c
case ssh.NativeClient:
return c.OutputWithPty(args)
}
return client.Output(args)
}
// FIXME: pull uname -m into common code to just do it once, like reading /etc/os-release
func (provisioner *s390xSUSEProvisioner) CompatibleWithHost() bool {
arch, err := provisioner.SSHCommand("uname -m")
return err == nil && provisioner.OsReleaseInfo.Id == provisioner.OsReleaseId &&
strings.Contains(arch, "s390x")
}
func (provisioner *s390xSUSEProvisioner) SetHostname(hostname string) error {
// we have to have SetHostname here as well to use the SUSE provisioner
// SSHCommand to add the tty allocation
if _, err := provisioner.SSHCommand(fmt.Sprintf(
"sudo hostname %s && echo %q | sudo tee /etc/hostname",
hostname,
hostname,
)); err != nil {
return err
}
// ubuntu/debian use 127.0.1.1 for non "localhost" loopback hostnames: https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution
if _, err := provisioner.SSHCommand(fmt.Sprintf(
"if grep -xq 127.0.1.1.* /etc/hosts; then sudo sed -i 's/^127.0.1.1.*/127.0.1.1 %s/g' /etc/hosts; else echo '127.0.1.1 %s' | sudo tee -a /etc/hosts; fi",
hostname,
hostname,
)); err != nil {
return err
}
return nil
}
func (provisioner *s390xSUSEProvisioner) Service(name string, action serviceaction.ServiceAction) error {
reloadDaemon := false
switch action {
case serviceaction.Start, serviceaction.Restart:
reloadDaemon = true
}
// systemd needs reloaded when config changes on disk; we cannot
// be sure exactly when it changes from the provisioner so
// we call a reload on every restart to be safe
if reloadDaemon {
if _, err := provisioner.SSHCommand("sudo systemctl daemon-reload"); err != nil {
return err
}
}
command := fmt.Sprintf("sudo systemctl %s %s", action.String(), name)
if _, err := provisioner.SSHCommand(command); err != nil {
return err
}
return nil
}
func (provisioner *s390xSUSEProvisioner) Package(name string, action pkgaction.PackageAction) error {
var packageAction string
switch action {
case pkgaction.Install:
packageAction = "install"
case pkgaction.Remove:
packageAction = "remove"
case pkgaction.Upgrade:
packageAction = "update"
}
command := fmt.Sprintf("sudo -E zypper -n %s %s", packageAction, name)
if _, err := provisioner.SSHCommand(command); err != nil {
return err
}
return nil
}
func (provisioner *s390xSUSEProvisioner) s390xSUSEInstallDockerPerEnv(cmd string) error {
if _, err := provisioner.SSHCommand(cmd); err != nil {
return fmt.Errorf("error installing docker: %s\n", err)
}
return nil
}
func (provisioner *s390xSUSEProvisioner) dockerDaemonResponding() bool {
if _, err := provisioner.SSHCommand("sudo docker version"); err != nil {
log.Warnf("Error getting SSH command to check if the daemon is up: %s", err)
return false
}
// The daemon is up if the command worked. Carry on.
return true
}
func (provisioner *s390xSUSEProvisioner) Provision(swarmOptions swarm.SwarmOptions, authOptions auth.AuthOptions, engineOptions engine.EngineOptions) error {
provisioner.SwarmOptions = swarmOptions
provisioner.AuthOptions = authOptions
provisioner.EngineOptions = engineOptions
// set default storage driver for suse
if provisioner.EngineOptions.StorageDriver == "" {
provisioner.EngineOptions.StorageDriver = "devicemapper"
}
if err := provisioner.SetHostname(provisioner.Driver.GetMachineName()); err != nil {
return err
}
for _, pkg := range provisioner.Packages {
log.Debugf("installing base package: name=%s", pkg)
if err := provisioner.Package(pkg, pkgaction.Install); err != nil {
return err
}
}
// update OS -- this is needed for libdevicemapper and the docker install
if _, err := provisioner.SSHCommand("sudo zypper -n update"); err != nil {
return err
}
s390xSUSEInstallCmd := os.Getenv("MACHINE_s390xSUSE_INSTALL_CMD")
if s390xSUSEInstallCmd != "" {
if err := provisioner.s390xSUSEInstallDockerPerEnv(s390xSUSEInstallCmd); err != nil {
return err
}
} else {
if err := installDockerGeneric(provisioner, engineOptions.InstallURL); err != nil {
return err
}
}
if err := makeDockerOptionsDir(provisioner); err != nil {
return err
}
provisioner.AuthOptions = setRemoteAuthOptions(provisioner)
if err := ConfigureAuth(provisioner); err != nil {
return err
}
if err := mcnutils.WaitFor(provisioner.dockerDaemonResponding); err != nil {
return err
}
if err := configureSwarm(provisioner, swarmOptions, provisioner.AuthOptions); err != nil {
return err
}
return nil
}
func (provisioner *s390xSUSEProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) {
var (
engineCfg bytes.Buffer
configPath = provisioner.DaemonOptionsFile
)
driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName())
provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel)
t, err := template.New("engineConfig").Parse(s3suseEngineConfigTemplate)
if err != nil {
return nil, err
}
engineConfigContext := EngineConfigContext{
DockerPort: dockerPort,
AuthOptions: provisioner.AuthOptions,
EngineOptions: provisioner.EngineOptions,
DockerOptionsDir: provisioner.DockerOptionsDir,
}
t.Execute(&engineCfg, engineConfigContext)
daemonOptsDir := configPath
return &DockerOptions{
EngineOptions: engineCfg.String(),
EngineOptionsPath: daemonOptsDir,
}, nil
}
package provisionAlso, add the following function to the end of /go/src/github.com/docker/machine/libmachine/provision/redhat.go:
import (
"bytes"
"fmt"
"os"
"strings"
"text/template"
"github.com/docker/machine/libmachine/auth"
"github.com/docker/machine/libmachine/drivers"
"github.com/docker/machine/libmachine/engine"
"github.com/docker/machine/libmachine/log"
"github.com/docker/machine/libmachine/mcnutils"
"github.com/docker/machine/libmachine/provision/pkgaction"
"github.com/docker/machine/libmachine/provision/serviceaction"
"github.com/docker/machine/libmachine/ssh"
"github.com/docker/machine/libmachine/swarm"
)
var (
s3redhatEngineConfigTemplate = `[Service]
ExecStart=/usr/bin/docker -d -H tcp://0.0.0.0:{{.DockerPort}} -H unix:///var/run/docker.sock --storage-driver {{.EngineOptions.StorageDriver}} --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}} {{ range .EngineOptions.Labels }}--label {{.}} {{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} {{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} {{ end }}
MountFlags=slave
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
Environment={{range .EngineOptions.Env}}{{ printf "%q" . }} {{end}}
`
)
func init() {
Register("RedHat-s390x", &RegisteredProvisioner{
New: News390xRedHatProvisioner,
})
}
func News390xRedHatProvisioner(d drivers.Driver) Provisioner {
return &s390xRedHatProvisioner{
GenericProvisioner: GenericProvisioner{
DockerOptionsDir: "/etc/docker",
DaemonOptionsFile: "/etc/systemd/system/docker.service",
OsReleaseId: "rhel",
Packages: []string{
"curl", "sudo",
},
Driver: d,
},
}
}
type s390xRedHatProvisioner struct {
GenericProvisioner
}
func (provisioner *s390xRedHatProvisioner) SSHCommand(args string) (string, error) {
client, err := drivers.GetSSHClientFromDriver(provisioner.Driver)
if err != nil {
return "", err
}
// redhat needs "-t" for tty allocation on ssh therefore we check for the
// external client and add as needed.
// Note: CentOS 7.0 needs multiple "-tt" to force tty allocation when ssh has
// no local tty.
switch c := client.(type) {
case ssh.ExternalClient:
c.BaseArgs = append(c.BaseArgs, "-tt")
client = c
case ssh.NativeClient:
return c.OutputWithPty(args)
}
return client.Output(args)
}
// FIXME: pull uname -m into common code to just do it once, like reading /etc/os-release
func (provisioner *s390xRedHatProvisioner) CompatibleWithHost() bool {
arch, err := provisioner.SSHCommand("uname -m")
return err == nil && provisioner.OsReleaseInfo.Id == provisioner.OsReleaseId &&
strings.Contains(arch, "s390x")
}
func (provisioner *s390xRedHatProvisioner) SetHostname(hostname string) error {
// we have to have SetHostname here as well to use the s390xRedHat provisioner
// SSHCommand to add the tty allocation
if _, err := provisioner.SSHCommand(fmt.Sprintf(
"sudo hostname %s && echo %q | sudo tee /etc/hostname",
hostname,
hostname,
)); err != nil {
return err
}
// ubuntu/debian use 127.0.1.1 for non "localhost" loopback hostnames: https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution
if _, err := provisioner.SSHCommand(fmt.Sprintf(
"if grep -xq 127.0.1.1.* /etc/hosts; then sudo sed -i 's/^127.0.1.1.*/127.0.1.1 %s/g' /etc/hosts; else echo '127.0.1.1 %s' | sudo tee -a /etc/hosts; fi",
hostname,
hostname,
)); err != nil {
return err
}
return nil
}
func (provisioner *s390xRedHatProvisioner) Service(name string, action serviceaction.ServiceAction) error {
reloadDaemon := false
switch action {
case serviceaction.Start, serviceaction.Restart:
reloadDaemon = true
}
// systemd needs reloaded when config changes on disk; we cannot
// be sure exactly when it changes from the provisioner so
// we call a reload on every restart to be safe
if reloadDaemon {
if _, err := provisioner.SSHCommand("sudo systemctl daemon-reload"); err != nil {
return err
}
}
command := fmt.Sprintf("sudo systemctl %s %s", action.String(), name)
if _, err := provisioner.SSHCommand(command); err != nil {
return err
}
return nil
}
func (provisioner *s390xRedHatProvisioner) Package(name string, action pkgaction.PackageAction) error {
var packageAction string
switch action {
case pkgaction.Install:
packageAction = "install"
case pkgaction.Remove:
packageAction = "remove"
case pkgaction.Upgrade:
packageAction = "upgrade"
}
command := fmt.Sprintf("sudo -E yum %s -y %s", packageAction, name)
if _, err := provisioner.SSHCommand(command); err != nil {
return err
}
return nil
}
func s390xRedHatInstallDockerPerEnv(p Provisioner, cmd string) error {
if output, err := p.SSHCommand(cmd); err != nil {
return fmt.Errorf("error installing docker: %s\n", output)
}
return nil
}
func (provisioner *s390xRedHatProvisioner) dockerDaemonResponding() bool {
if _, err := provisioner.SSHCommand("sudo docker version"); err != nil {
log.Warnf("Error getting SSH command to check if the daemon is up: %s", err)
return false
}
// The daemon is up if the command worked. Carry on.
return true
}
func (provisioner *s390xRedHatProvisioner) Provision(swarmOptions swarm.SwarmOptions, authOptions auth.AuthOptions, engineOptions engine.EngineOptions) error {
provisioner.SwarmOptions = swarmOptions
provisioner.AuthOptions = authOptions
provisioner.EngineOptions = engineOptions
// set default storage driver for redhat
if provisioner.EngineOptions.StorageDriver == "" {
provisioner.EngineOptions.StorageDriver = "devicemapper"
}
if err := provisioner.SetHostname(provisioner.Driver.GetMachineName()); err != nil {
return err
}
for _, pkg := range provisioner.Packages {
log.Debugf("installing base package: name=%s", pkg)
if err := provisioner.Package(pkg, pkgaction.Install); err != nil {
return err
}
}
// update OS -- this is needed for libdevicemapper and the docker install
if _, err := provisioner.SSHCommand("sudo yum -y update"); err != nil {
return err
}
// install docker
s390xRedHatInstallCmd := os.Getenv("MACHINE_s390xREDHAT_INSTALL_CMD")
if s390xRedHatInstallCmd != "" {
if err := s390xRedHatInstallDockerPerEnv(provisioner, s390xRedHatInstallCmd); err != nil {
return err
}
} else {
if err := installDockerGeneric(provisioner, engineOptions.InstallURL); err != nil {
return err
}
}
if err := makeDockerOptionsDir(provisioner); err != nil {
return err
}
provisioner.AuthOptions = setRemoteAuthOptions(provisioner)
if err := ConfigureAuth(provisioner); err != nil {
return err
}
if err := mcnutils.WaitFor(provisioner.dockerDaemonResponding); err != nil {
return err
}
if err := configureSwarm(provisioner, swarmOptions, provisioner.AuthOptions); err != nil {
return err
}
return nil
}
func (provisioner *s390xRedHatProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) {
var (
engineCfg bytes.Buffer
configPath = provisioner.DaemonOptionsFile
)
driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName())
provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel)
// systemd / redhat will not load options if they are on newlines
// instead, it just continues with a different set of options; yeah...
t, err := template.New("engineConfig").Parse(s3redhatEngineConfigTemplate)
if err != nil {
return nil, err
}
engineConfigContext := EngineConfigContext{
DockerPort: dockerPort,
AuthOptions: provisioner.AuthOptions,
EngineOptions: provisioner.EngineOptions,
DockerOptionsDir: provisioner.DockerOptionsDir,
}
t.Execute(&engineCfg, engineConfigContext)
daemonOptsDir := configPath
return &DockerOptions{
EngineOptions: engineCfg.String(),
EngineOptionsPath: daemonOptsDir,
}, nil
}
// FIXME: pull uname -m into common code to just do it once, like reading /etc/os-releaseIn the same file, redhat.go, add another line into the import statement right at the beginning, just like there is a line with "fmt":
func (provisioner *RedHatProvisioner) CompatibleWithHost() bool {
arch, err := provisioner.SSHCommand("uname -m")
return err == nil && provisioner.OsReleaseInfo.Id == provisioner.OsReleaseId &&
!strings.Contains(arch, "s390x")
}
"strings"All the above changes add s390x support for both RHEL and SLES and make sure this does not collide with the present RHEL/x86 support. Instead of going to a predefined package repository and downloading some files, our own installation method is used. Both files are adaptions from the original way of installing RHEL. The code assumes your hosts are hooked up to a yum/zypper repository.
We need one quirk in our gcc-based build environment: the Azure stuff seems in flux (see also here), so we want to use the dependencies provided with docker machine at this time:
export GOROOT=/go/src/github.com/docker/machine/Godeps/_workspace/The build is then kicked off via
cd /go/src/github.com/docker/machine
make build
bin/ will get a set of binaries which make up docker-machine. Be aware to transport libgcc_s.so.1 and libgo.so.7 when copying this to another system, and add the path of these files into the environment variable LD_LIBRARY_PATH.
Use the technique from Image Decomposition to generate a small container with all the necessary files in it (have ENTRYPOINT refer to docker-machine, and make sure you mount ~/.docker or at least ~/.docker/machine to the container, since that is where state information is put).
How do we use machine? The documentation points out four options we can use to specify which Linux host we want to convert into a Docker host. machine uses public key authentication to access the existing host. If your target host does not carry the public ssh key, you need to add it, e.g. via ssh-copy-id. For most purposes, the defaults as shown on the documentation page are fine, thus we only need to specify --generic-ip-address.
When installing docker on the Linux host, machine will try to detect the flavor of the host OS. Depending on the specific distribution, a suitable installation method is used: e.g. for a Red Hat system, docker is installed through a docker-originating yum repo server. For an Ubuntu system, an installation script is used. This script is downloaded from http://get.docker.com/ and simply executed.
However, today's machine code only deals with installation on x86. The code we added previous provides an extension to install docker on Linux on z systems. For both SLES and RHEL systems, the installation script method is used. Since http://get.docker.com/ only considersx86, we can use the parameter --engine-install-url to define the installation script URL.
Alternatively, you could change the DNS resolution (via /etc/hosts) to have get.docker.com resolved to your own web server.
The installation script served by the web server can look like this:
#!/bin/shThe changes to docker machine as introduced by the code above also allow for yet another alternative: instead of setting a webserver up with the script, it can be provided in shell variables: setting MACHINE_s390xSUSE_INSTALL_CMD respectively MACHINE_s390xREDHAT_INSTALL_CMD makes docker-machine use exactly that code. The parameter --engine-install-url is not needed anymore in that case. So e.g. you can do a:
SUSE_URL=ftp://ftp.unicamp.br/pub/linuxpatch/s390x/suse/sles12/docker/docker-1.8.2-sles12-20150930.tar.gz
SUSE_FILE_BASE=docker-1.8.2-sles12-20150930
REDHAT_URL=ftp://ftp.unicamp.br/pub/linuxpatch/s390x/redhat/rhel7.1/docker/docker-1.8.2-rhel7.1-20150929.tar.gz
REDHAT_FILE_BASE=docker-1.8.2-rhel7.1-20150929
function set_redhat_install() {
URL=$REDHAT_URL
FILE_BASE=$REDHAT_FILE_BASE
yum -y install wget
}
function set_suse_install() {
URL=$SUSE_URL
FILE_BASE=$SUSE_FILE_BASE
zypper -n install wget
}
function do_install() {
cd /tmp &&
wget $URL &&
tar xzf ${FILE_BASE}.tar.gz &&
cp ${FILE_BASE}/docker /usr/bin &&
rm -fr /tmp/${FILE_BASE}.tar.gz /tmp/$FILE_BASE
}
id=`grep "^ID=" /etc/os-release | cut -d'"' -f2`
if [ $id = "sles" ]
then
set_suse_install
else
set_redhat_install
fi
do_install
export MACHINE_s390xSUSE_INSTALL_CMD="wget http://r1745035/docker-1.8.2-sles12-20150930.tar.gz;tar xzf docker-1.8.2-sles12-20150930.tar.gz;cp docker-1.8.2-sles12-20150930/docker /usr/bin;rm -fr docker-1.8.2-sles12-20150930 docker-1.8.2-sles12-20150930.tar.gz"This snippet is a simplified version of the installation script (assuming wget is already there) and makes machine download the docker binaries from a webserver (r1745035 in that case).
After all this setup work, docker machine can creat docker hosts on s390x:
[root@r1745035 ~]# docker-machine create --engine-install-url http://r1745035/ -d generic --generic-ip-address r1745034 node034We will follow the last advice of the output:
Running pre-create checks...
Creating machine...
Waiting for machine to be running, this may take a few minutes...
Machine is running, waiting for SSH to be available...
Detecting operating system of created instance...
Provisioning created instance...
Copying certs to the local machine directory...
Copying certs to the remote machine...
Setting Docker configuration on the remote daemon...
To see how to connect Docker to this machine, run: docker-machine env node034
[root@r1745035 ~]# docker-machine env node034Ok, we can do that:
export DOCKER_TLS_VERIFY="1"
export DOCKER_HOST="tcp://r1745034:2376"
export DOCKER_CERT_PATH="/root/.docker/machine/certs"
export DOCKER_MACHINE_NAME="node034"
# Run this command to configure your shell:
# eval "$(docker-machine env node034)"
[root@r1745035 ~]# eval "$(docker-machine env node034)"Then docker will refer to that newly deployed docker host:
[root@r1745035 ~]# docker info | grep ^NamePlease note that this code and installation script is not final code. In particular, it needs to be looked at when distributions provide docker as part of their packages -- that will determine how the code going into machine will look like eventually. Yet the current state ought to be good enough to get started.
Name: node034
No comments:
Post a Comment
Note: Only a member of this blog may post a comment.