integration-cli/docker_api_swarm_test.go

   1 // +build !windows
   2
   3 package main
   4
   5 import (
   6         "encoding/json"
   7         "fmt"
   8         "io/ioutil"
   9         "net"
  10         "net/http"
  11         "net/url"
  12         "os"
  13         "path/filepath"
  14         "strings"
  15         "sync"
  16         "time"
  17
  18         "github.com/cloudflare/cfssl/csr"
  19         "github.com/cloudflare/cfssl/helpers"
  20         "github.com/cloudflare/cfssl/initca"
  21         "github.com/docker/docker/api/types"
  22         "github.com/docker/docker/api/types/container"
  23         "github.com/docker/docker/api/types/swarm"
  24         "github.com/docker/docker/integration-cli/checker"
  25         "github.com/docker/docker/integration-cli/daemon"
  26         "github.com/docker/swarmkit/ca"
  27         "github.com/go-check/check"
  28 )
  29
  30 var defaultReconciliationTimeout = 30 * time.Second
  31
  32 func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) {
  33         // todo: should find a better way to verify that components are running than /info
  34         d1 := s.AddDaemon(c, true, true)
  35         info, err := d1.SwarmInfo()
  36         c.Assert(err, checker.IsNil)
  37         c.Assert(info.ControlAvailable, checker.True)
  38         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  39         c.Assert(info.Cluster.RootRotationInProgress, checker.False)
  40
  41         d2 := s.AddDaemon(c, true, false)
  42         info, err = d2.SwarmInfo()
  43         c.Assert(err, checker.IsNil)
  44         c.Assert(info.ControlAvailable, checker.False)
  45         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  46
  47         // Leaving cluster
  48         c.Assert(d2.Leave(false), checker.IsNil)
  49
  50         info, err = d2.SwarmInfo()
  51         c.Assert(err, checker.IsNil)
  52         c.Assert(info.ControlAvailable, checker.False)
  53         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  54
  55         c.Assert(d2.Join(swarm.JoinRequest{JoinToken: d1.JoinTokens(c).Worker, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil)
  56
  57         info, err = d2.SwarmInfo()
  58         c.Assert(err, checker.IsNil)
  59         c.Assert(info.ControlAvailable, checker.False)
  60         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  61
  62         // Current state restoring after restarts
  63         d1.Stop(c)
  64         d2.Stop(c)
  65
  66         d1.Start(c)
  67         d2.Start(c)
  68
  69         info, err = d1.SwarmInfo()
  70         c.Assert(err, checker.IsNil)
  71         c.Assert(info.ControlAvailable, checker.True)
  72         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  73
  74         info, err = d2.SwarmInfo()
  75         c.Assert(err, checker.IsNil)
  76         c.Assert(info.ControlAvailable, checker.False)
  77         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  78 }
  79
  80 func (s *DockerSwarmSuite) TestAPISwarmJoinToken(c *check.C) {
  81         d1 := s.AddDaemon(c, false, false)
  82         c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil)
  83
  84         // todo: error message differs depending if some components of token are valid
  85
  86         d2 := s.AddDaemon(c, false, false)
  87         err := d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.ListenAddr}})
  88         c.Assert(err, checker.NotNil)
  89         c.Assert(err.Error(), checker.Contains, "join token is necessary")
  90         info, err := d2.SwarmInfo()
  91         c.Assert(err, checker.IsNil)
  92         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  93
  94         err = d2.Join(swarm.JoinRequest{JoinToken: "foobaz", RemoteAddrs: []string{d1.ListenAddr}})
  95         c.Assert(err, checker.NotNil)
  96         c.Assert(err.Error(), checker.Contains, "invalid join token")
  97         info, err = d2.SwarmInfo()
  98         c.Assert(err, checker.IsNil)
  99         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
 100
 101         workerToken := d1.JoinTokens(c).Worker
 102
 103         c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil)
 104         info, err = d2.SwarmInfo()
 105         c.Assert(err, checker.IsNil)
 106         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
 107         c.Assert(d2.Leave(false), checker.IsNil)
 108         info, err = d2.SwarmInfo()
 109         c.Assert(err, checker.IsNil)
 110         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
 111
 112         // change tokens
 113         d1.RotateTokens(c)
 114
 115         err = d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}})
 116         c.Assert(err, checker.NotNil)
 117         c.Assert(err.Error(), checker.Contains, "join token is necessary")
 118         info, err = d2.SwarmInfo()
 119         c.Assert(err, checker.IsNil)
 120         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
 121
 122         workerToken = d1.JoinTokens(c).Worker
 123
 124         c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil)
 125         info, err = d2.SwarmInfo()
 126         c.Assert(err, checker.IsNil)
 127         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
 128         c.Assert(d2.Leave(false), checker.IsNil)
 129         info, err = d2.SwarmInfo()
 130         c.Assert(err, checker.IsNil)
 131         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
 132
 133         // change spec, don't change tokens
 134         d1.UpdateSwarm(c, func(s *swarm.Spec) {})
 135
 136         err = d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.ListenAddr}})
 137         c.Assert(err, checker.NotNil)
 138         c.Assert(err.Error(), checker.Contains, "join token is necessary")
 139         info, err = d2.SwarmInfo()
 140         c.Assert(err, checker.IsNil)
 141         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
 142
 143         c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil)
 144         info, err = d2.SwarmInfo()
 145         c.Assert(err, checker.IsNil)
 146         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
 147         c.Assert(d2.Leave(false), checker.IsNil)
 148         info, err = d2.SwarmInfo()
 149         c.Assert(err, checker.IsNil)
 150         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
 151 }
 152
 153 func (s *DockerSwarmSuite) TestUpdateSwarmAddExternalCA(c *check.C) {
 154         d1 := s.AddDaemon(c, false, false)
 155         c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil)
 156         d1.UpdateSwarm(c, func(s *swarm.Spec) {
 157                 s.CAConfig.ExternalCAs = []*swarm.ExternalCA{
 158                         {
 159                                 Protocol: swarm.ExternalCAProtocolCFSSL,
 160                                 URL:      "https://thishasnoca.org",
 161                         },
 162                         {
 163                                 Protocol: swarm.ExternalCAProtocolCFSSL,
 164                                 URL:      "https://thishasacacert.org",
 165                                 CACert:   "cacert",
 166                         },
 167                 }
 168         })
 169         info, err := d1.SwarmInfo()
 170         c.Assert(err, checker.IsNil)
 171         c.Assert(info.Cluster.Spec.CAConfig.ExternalCAs, checker.HasLen, 2)
 172         c.Assert(info.Cluster.Spec.CAConfig.ExternalCAs[0].CACert, checker.Equals, "")
 173         c.Assert(info.Cluster.Spec.CAConfig.ExternalCAs[1].CACert, checker.Equals, "cacert")
 174 }
 175
 176 func (s *DockerSwarmSuite) TestAPISwarmCAHash(c *check.C) {
 177         d1 := s.AddDaemon(c, true, true)
 178         d2 := s.AddDaemon(c, false, false)
 179         splitToken := strings.Split(d1.JoinTokens(c).Worker, "-")
 180         splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e"
 181         replacementToken := strings.Join(splitToken, "-")
 182         err := d2.Join(swarm.JoinRequest{JoinToken: replacementToken, RemoteAddrs: []string{d1.ListenAddr}})
 183         c.Assert(err, checker.NotNil)
 184         c.Assert(err.Error(), checker.Contains, "remote CA does not match fingerprint")
 185 }
 186
 187 func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) {
 188         d1 := s.AddDaemon(c, false, false)
 189         c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil)
 190         d2 := s.AddDaemon(c, true, false)
 191
 192         info, err := d2.SwarmInfo()
 193         c.Assert(err, checker.IsNil)
 194         c.Assert(info.ControlAvailable, checker.False)
 195         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
 196
 197         d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) {
 198                 n.Spec.Role = swarm.NodeRoleManager
 199         })
 200
 201         waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.True)
 202
 203         d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) {
 204                 n.Spec.Role = swarm.NodeRoleWorker
 205         })
 206
 207         waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.False)
 208
 209         // Wait for the role to change to worker in the cert. This is partially
 210         // done because it's something worth testing in its own right, and
 211         // partially because changing the role from manager to worker and then
 212         // back to manager quickly might cause the node to pause for awhile
 213         // while waiting for the role to change to worker, and the test can
 214         // time out during this interval.
 215         waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
 216                 certBytes, err := ioutil.ReadFile(filepath.Join(d2.Folder, "root", "swarm", "certificates", "swarm-node.crt"))
 217                 if err != nil {
 218                         return "", check.Commentf("error: %v", err)
 219                 }
 220                 certs, err := helpers.ParseCertificatesPEM(certBytes)
 221                 if err == nil && len(certs) > 0 && len(certs[0].Subject.OrganizationalUnit) > 0 {
 222                         return certs[0].Subject.OrganizationalUnit[0], nil
 223                 }
 224                 return "", check.Commentf("could not get organizational unit from certificate")
 225         }, checker.Equals, "swarm-worker")
 226
 227         // Demoting last node should fail
 228         node := d1.GetNode(c, d1.NodeID)
 229         node.Spec.Role = swarm.NodeRoleWorker
 230         url := fmt.Sprintf("/nodes/%s/update?version=%d", node.ID, node.Version.Index)
 231         status, out, err := d1.SockRequest("POST", url, node.Spec)
 232         c.Assert(err, checker.IsNil)
 233         c.Assert(status, checker.Equals, http.StatusBadRequest, check.Commentf("output: %q", string(out)))
 234         // The warning specific to demoting the last manager is best-effort and
 235         // won't appear until the Role field of the demoted manager has been
 236         // updated.
 237         // Yes, I know this looks silly, but checker.Matches is broken, since
 238         // it anchors the regexp contrary to the documentation, and this makes
 239         // it impossible to match something that includes a line break.
 240         if !strings.Contains(string(out), "last manager of the swarm") {
 241                 c.Assert(string(out), checker.Contains, "this would result in a loss of quorum")
 242         }
 243         info, err = d1.SwarmInfo()
 244         c.Assert(err, checker.IsNil)
 245         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
 246         c.Assert(info.ControlAvailable, checker.True)
 247
 248         // Promote already demoted node
 249         d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) {
 250                 n.Spec.Role = swarm.NodeRoleManager
 251         })
 252
 253         waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.True)
 254 }
 255
 256 func (s *DockerSwarmSuite) TestAPISwarmLeaderProxy(c *check.C) {
 257         // add three managers, one of these is leader
 258         d1 := s.AddDaemon(c, true, true)
 259         d2 := s.AddDaemon(c, true, true)
 260         d3 := s.AddDaemon(c, true, true)
 261
 262         // start a service by hitting each of the 3 managers
 263         d1.CreateService(c, simpleTestService, func(s *swarm.Service) {
 264                 s.Spec.Name = "test1"
 265         })
 266         d2.CreateService(c, simpleTestService, func(s *swarm.Service) {
 267                 s.Spec.Name = "test2"
 268         })
 269         d3.CreateService(c, simpleTestService, func(s *swarm.Service) {
 270                 s.Spec.Name = "test3"
 271         })
 272
 273         // 3 services should be started now, because the requests were proxied to leader
 274         // query each node and make sure it returns 3 services
 275         for _, d := range []*daemon.Swarm{d1, d2, d3} {
 276                 services := d.ListServices(c)
 277                 c.Assert(services, checker.HasLen, 3)
 278         }
 279 }
 280
 281 func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) {
 282         // Create 3 nodes
 283         d1 := s.AddDaemon(c, true, true)
 284         d2 := s.AddDaemon(c, true, true)
 285         d3 := s.AddDaemon(c, true, true)
 286
 287         // assert that the first node we made is the leader, and the other two are followers
 288         c.Assert(d1.GetNode(c, d1.NodeID).ManagerStatus.Leader, checker.True)
 289         c.Assert(d1.GetNode(c, d2.NodeID).ManagerStatus.Leader, checker.False)
 290         c.Assert(d1.GetNode(c, d3.NodeID).ManagerStatus.Leader, checker.False)
 291
 292         d1.Stop(c)
 293
 294         var (
 295                 leader    *daemon.Swarm   // keep track of leader
 296                 followers []*daemon.Swarm // keep track of followers
 297         )
 298         checkLeader := func(nodes ...*daemon.Swarm) checkF {
 299                 return func(c *check.C) (interface{}, check.CommentInterface) {
 300                         // clear these out before each run
 301                         leader = nil
 302                         followers = nil
 303                         for _, d := range nodes {
 304                                 if d.GetNode(c, d.NodeID).ManagerStatus.Leader {
 305                                         leader = d
 306                                 } else {
 307                                         followers = append(followers, d)
 308                                 }
 309                         }
 310
 311                         if leader == nil {
 312                                 return false, check.Commentf("no leader elected")
 313                         }
 314
 315                         return true, check.Commentf("elected %v", leader.ID())
 316                 }
 317         }
 318
 319         // wait for an election to occur
 320         waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d2, d3), checker.True)
 321
 322         // assert that we have a new leader
 323         c.Assert(leader, checker.NotNil)
 324
 325         // Keep track of the current leader, since we want that to be chosen.
 326         stableleader := leader
 327
 328         // add the d1, the initial leader, back
 329         d1.Start(c)
 330
 331         // TODO(stevvooe): may need to wait for rejoin here
 332
 333         // wait for possible election
 334         waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d1, d2, d3), checker.True)
 335         // pick out the leader and the followers again
 336
 337         // verify that we still only have 1 leader and 2 followers
 338         c.Assert(leader, checker.NotNil)
 339         c.Assert(followers, checker.HasLen, 2)
 340         // and that after we added d1 back, the leader hasn't changed
 341         c.Assert(leader.NodeID, checker.Equals, stableleader.NodeID)
 342 }
 343
 344 func (s *DockerSwarmSuite) TestAPISwarmRaftQuorum(c *check.C) {
 345         d1 := s.AddDaemon(c, true, true)
 346         d2 := s.AddDaemon(c, true, true)
 347         d3 := s.AddDaemon(c, true, true)
 348
 349         d1.CreateService(c, simpleTestService)
 350
 351         d2.Stop(c)
 352
 353         // make sure there is a leader
 354         waitAndAssert(c, defaultReconciliationTimeout, d1.CheckLeader, checker.IsNil)
 355
 356         d1.CreateService(c, simpleTestService, func(s *swarm.Service) {
 357                 s.Spec.Name = "top1"
 358         })
 359
 360         d3.Stop(c)
 361
 362         var service swarm.Service
 363         simpleTestService(&service)
 364         service.Spec.Name = "top2"
 365         status, out, err := d1.SockRequest("POST", "/services/create", service.Spec)
 366         c.Assert(err, checker.IsNil)
 367         c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("deadline exceeded", string(out)))
 368
 369         d2.Start(c)
 370
 371         // make sure there is a leader
 372         waitAndAssert(c, defaultReconciliationTimeout, d1.CheckLeader, checker.IsNil)
 373
 374         d1.CreateService(c, simpleTestService, func(s *swarm.Service) {
 375                 s.Spec.Name = "top3"
 376         })
 377 }
 378
 379 func (s *DockerSwarmSuite) TestAPISwarmLeaveRemovesContainer(c *check.C) {
 380         d := s.AddDaemon(c, true, true)
 381
 382         instances := 2
 383         d.CreateService(c, simpleTestService, setInstances(instances))
 384
 385         id, err := d.Cmd("run", "-d", "busybox", "top")
 386         c.Assert(err, checker.IsNil)
 387         id = strings.TrimSpace(id)
 388
 389         waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances+1)
 390
 391         c.Assert(d.Leave(false), checker.NotNil)
 392         c.Assert(d.Leave(true), checker.IsNil)
 393
 394         waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1)
 395
 396         id2, err := d.Cmd("ps", "-q")
 397         c.Assert(err, checker.IsNil)
 398         c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2))
 399 }
 400
 401 // #23629
 402 func (s *DockerSwarmSuite) TestAPISwarmLeaveOnPendingJoin(c *check.C) {
 403         testRequires(c, Network)
 404         s.AddDaemon(c, true, true)
 405         d2 := s.AddDaemon(c, false, false)
 406
 407         id, err := d2.Cmd("run", "-d", "busybox", "top")
 408         c.Assert(err, checker.IsNil)
 409         id = strings.TrimSpace(id)
 410
 411         err = d2.Join(swarm.JoinRequest{
 412                 RemoteAddrs: []string{"123.123.123.123:1234"},
 413         })
 414         c.Assert(err, check.NotNil)
 415         c.Assert(err.Error(), checker.Contains, "Timeout was reached")
 416
 417         info, err := d2.SwarmInfo()
 418         c.Assert(err, checker.IsNil)
 419         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStatePending)
 420
 421         c.Assert(d2.Leave(true), checker.IsNil)
 422
 423         waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 1)
 424
 425         id2, err := d2.Cmd("ps", "-q")
 426         c.Assert(err, checker.IsNil)
 427         c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2))
 428 }
 429
 430 // #23705
 431 func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) {
 432         testRequires(c, Network)
 433         d := s.AddDaemon(c, false, false)
 434         err := d.Join(swarm.JoinRequest{
 435                 RemoteAddrs: []string{"123.123.123.123:1234"},
 436         })
 437         c.Assert(err, check.NotNil)
 438         c.Assert(err.Error(), checker.Contains, "Timeout was reached")
 439
 440         waitAndAssert(c, defaultReconciliationTimeout, d.CheckLocalNodeState, checker.Equals, swarm.LocalNodeStatePending)
 441
 442         d.Stop(c)
 443         d.Start(c)
 444
 445         info, err := d.SwarmInfo()
 446         c.Assert(err, checker.IsNil)
 447         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
 448 }
 449
 450 func (s *DockerSwarmSuite) TestAPISwarmManagerRestore(c *check.C) {
 451         d1 := s.AddDaemon(c, true, true)
 452
 453         instances := 2
 454         id := d1.CreateService(c, simpleTestService, setInstances(instances))
 455
 456         d1.GetService(c, id)
 457         d1.Stop(c)
 458         d1.Start(c)
 459         d1.GetService(c, id)
 460
 461         d2 := s.AddDaemon(c, true, true)
 462         d2.GetService(c, id)
 463         d2.Stop(c)
 464         d2.Start(c)
 465         d2.GetService(c, id)
 466
 467         d3 := s.AddDaemon(c, true, true)
 468         d3.GetService(c, id)
 469         d3.Stop(c)
 470         d3.Start(c)
 471         d3.GetService(c, id)
 472
 473         d3.Kill()
 474         time.Sleep(1 * time.Second) // time to handle signal
 475         d3.Start(c)
 476         d3.GetService(c, id)
 477 }
 478
 479 func (s *DockerSwarmSuite) TestAPISwarmScaleNoRollingUpdate(c *check.C) {
 480         d := s.AddDaemon(c, true, true)
 481
 482         instances := 2
 483         id := d.CreateService(c, simpleTestService, setInstances(instances))
 484
 485         waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
 486         containers := d.ActiveContainers()
 487         instances = 4
 488         d.UpdateService(c, d.GetService(c, id), setInstances(instances))
 489         waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
 490         containers2 := d.ActiveContainers()
 491
 492 loop0:
 493         for _, c1 := range containers {
 494                 for _, c2 := range containers2 {
 495                         if c1 == c2 {
 496                                 continue loop0
 497                         }
 498                 }
 499                 c.Errorf("container %v not found in new set %#v", c1, containers2)
 500         }
 501 }
 502
 503 func (s *DockerSwarmSuite) TestAPISwarmInvalidAddress(c *check.C) {
 504         d := s.AddDaemon(c, false, false)
 505         req := swarm.InitRequest{
 506                 ListenAddr: "",
 507         }
 508         status, _, err := d.SockRequest("POST", "/swarm/init", req)
 509         c.Assert(err, checker.IsNil)
 510         c.Assert(status, checker.Equals, http.StatusBadRequest)
 511
 512         req2 := swarm.JoinRequest{
 513                 ListenAddr:  "0.0.0.0:2377",
 514                 RemoteAddrs: []string{""},
 515         }
 516         status, _, err = d.SockRequest("POST", "/swarm/join", req2)
 517         c.Assert(err, checker.IsNil)
 518         c.Assert(status, checker.Equals, http.StatusBadRequest)
 519 }
 520
 521 func (s *DockerSwarmSuite) TestAPISwarmForceNewCluster(c *check.C) {
 522         d1 := s.AddDaemon(c, true, true)
 523         d2 := s.AddDaemon(c, true, true)
 524
 525         instances := 2
 526         id := d1.CreateService(c, simpleTestService, setInstances(instances))
 527         waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount), checker.Equals, instances)
 528
 529         // drain d2, all containers should move to d1
 530         d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) {
 531                 n.Spec.Availability = swarm.NodeAvailabilityDrain
 532         })
 533         waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances)
 534         waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 0)
 535
 536         d2.Stop(c)
 537
 538         c.Assert(d1.Init(swarm.InitRequest{
 539                 ForceNewCluster: true,
 540                 Spec:            swarm.Spec{},
 541         }), checker.IsNil)
 542
 543         waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances)
 544
 545         d3 := s.AddDaemon(c, true, true)
 546         info, err := d3.SwarmInfo()
 547         c.Assert(err, checker.IsNil)
 548         c.Assert(info.ControlAvailable, checker.True)
 549         c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
 550
 551         instances = 4
 552         d3.UpdateService(c, d3.GetService(c, id), setInstances(instances))
 553
 554         waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances)
 555 }
 556
 557 func simpleTestService(s *swarm.Service) {
 558         ureplicas := uint64(1)
 559         restartDelay := time.Duration(100 * time.Millisecond)
 560
 561         s.Spec = swarm.ServiceSpec{
 562                 TaskTemplate: swarm.TaskSpec{
 563                         ContainerSpec: swarm.ContainerSpec{
 564                                 Image:   "busybox:latest",
 565                                 Command: []string{"/bin/top"},
 566                         },
 567                         RestartPolicy: &swarm.RestartPolicy{
 568                                 Delay: &restartDelay,
 569                         },
 570                 },
 571                 Mode: swarm.ServiceMode{
 572                         Replicated: &swarm.ReplicatedService{
 573                                 Replicas: &ureplicas,
 574                         },
 575                 },
 576         }
 577         s.Spec.Name = "top"
 578 }
 579
 580 func serviceForUpdate(s *swarm.Service) {
 581         ureplicas := uint64(1)
 582         restartDelay := time.Duration(100 * time.Millisecond)
 583
 584         s.Spec = swarm.ServiceSpec{
 585                 TaskTemplate: swarm.TaskSpec{
 586                         ContainerSpec: swarm.ContainerSpec{
 587                                 Image:   "busybox:latest",
 588                                 Command: []string{"/bin/top"},
 589                         },
 590                         RestartPolicy: &swarm.RestartPolicy{
 591                                 Delay: &restartDelay,
 592                         },
 593                 },
 594                 Mode: swarm.ServiceMode{
 595                         Replicated: &swarm.ReplicatedService{
 596                                 Replicas: &ureplicas,
 597                         },
 598                 },
 599                 UpdateConfig: &swarm.UpdateConfig{
 600                         Parallelism:   2,
 601                         Delay:         4 * time.Second,
 602                         FailureAction: swarm.UpdateFailureActionContinue,
 603                 },
 604                 RollbackConfig: &swarm.UpdateConfig{
 605                         Parallelism:   3,
 606                         Delay:         4 * time.Second,
 607                         FailureAction: swarm.UpdateFailureActionContinue,
 608                 },
 609         }
 610         s.Spec.Name = "updatetest"
 611 }
 612
 613 func setInstances(replicas int) daemon.ServiceConstructor {
 614         ureplicas := uint64(replicas)
 615         return func(s *swarm.Service) {
 616                 s.Spec.Mode = swarm.ServiceMode{
 617                         Replicated: &swarm.ReplicatedService{
 618                                 Replicas: &ureplicas,
 619                         },
 620                 }
 621         }
 622 }
 623
 624 func setUpdateOrder(order string) daemon.ServiceConstructor {
 625         return func(s *swarm.Service) {
 626                 if s.Spec.UpdateConfig == nil {
 627                         s.Spec.UpdateConfig = &swarm.UpdateConfig{}
 628                 }
 629                 s.Spec.UpdateConfig.Order = order
 630         }
 631 }
 632
 633 func setRollbackOrder(order string) daemon.ServiceConstructor {
 634         return func(s *swarm.Service) {
 635                 if s.Spec.RollbackConfig == nil {
 636                         s.Spec.RollbackConfig = &swarm.UpdateConfig{}
 637                 }
 638                 s.Spec.RollbackConfig.Order = order
 639         }
 640 }
 641
 642 func setImage(image string) daemon.ServiceConstructor {
 643         return func(s *swarm.Service) {
 644                 s.Spec.TaskTemplate.ContainerSpec.Image = image
 645         }
 646 }
 647
 648 func setFailureAction(failureAction string) daemon.ServiceConstructor {
 649         return func(s *swarm.Service) {
 650                 s.Spec.UpdateConfig.FailureAction = failureAction
 651         }
 652 }
 653
 654 func setMaxFailureRatio(maxFailureRatio float32) daemon.ServiceConstructor {
 655         return func(s *swarm.Service) {
 656                 s.Spec.UpdateConfig.MaxFailureRatio = maxFailureRatio
 657         }
 658 }
 659
 660 func setParallelism(parallelism uint64) daemon.ServiceConstructor {
 661         return func(s *swarm.Service) {
 662                 s.Spec.UpdateConfig.Parallelism = parallelism
 663         }
 664 }
 665
 666 func setConstraints(constraints []string) daemon.ServiceConstructor {
 667         return func(s *swarm.Service) {
 668                 if s.Spec.TaskTemplate.Placement == nil {
 669                         s.Spec.TaskTemplate.Placement = &swarm.Placement{}
 670                 }
 671                 s.Spec.TaskTemplate.Placement.Constraints = constraints
 672         }
 673 }
 674
 675 func setPlacementPrefs(prefs []swarm.PlacementPreference) daemon.ServiceConstructor {
 676         return func(s *swarm.Service) {
 677                 if s.Spec.TaskTemplate.Placement == nil {
 678                         s.Spec.TaskTemplate.Placement = &swarm.Placement{}
 679                 }
 680                 s.Spec.TaskTemplate.Placement.Preferences = prefs
 681         }
 682 }
 683
 684 func setGlobalMode(s *swarm.Service) {
 685         s.Spec.Mode = swarm.ServiceMode{
 686                 Global: &swarm.GlobalService{},
 687         }
 688 }
 689
 690 func checkClusterHealth(c *check.C, cl []*daemon.Swarm, managerCount, workerCount int) {
 691         var totalMCount, totalWCount int
 692
 693         for _, d := range cl {
 694                 var (
 695                         info swarm.Info
 696                         err  error
 697                 )
 698
 699                 // check info in a waitAndAssert, because if the cluster doesn't have a leader, `info` will return an error
 700                 checkInfo := func(c *check.C) (interface{}, check.CommentInterface) {
 701                         info, err = d.SwarmInfo()
 702                         return err, check.Commentf("cluster not ready in time")
 703                 }
 704                 waitAndAssert(c, defaultReconciliationTimeout, checkInfo, checker.IsNil)
 705                 if !info.ControlAvailable {
 706                         totalWCount++
 707                         continue
 708                 }
 709
 710                 var leaderFound bool
 711                 totalMCount++
 712                 var mCount, wCount int
 713
 714                 for _, n := range d.ListNodes(c) {
 715                         waitReady := func(c *check.C) (interface{}, check.CommentInterface) {
 716                                 if n.Status.State == swarm.NodeStateReady {
 717                                         return true, nil
 718                                 }
 719                                 nn := d.GetNode(c, n.ID)
 720                                 n = *nn
 721                                 return n.Status.State == swarm.NodeStateReady, check.Commentf("state of node %s, reported by %s", n.ID, d.Info.NodeID)
 722                         }
 723                         waitAndAssert(c, defaultReconciliationTimeout, waitReady, checker.True)
 724
 725                         waitActive := func(c *check.C) (interface{}, check.CommentInterface) {
 726                                 if n.Spec.Availability == swarm.NodeAvailabilityActive {
 727                                         return true, nil
 728                                 }
 729                                 nn := d.GetNode(c, n.ID)
 730                                 n = *nn
 731                                 return n.Spec.Availability == swarm.NodeAvailabilityActive, check.Commentf("availability of node %s, reported by %s", n.ID, d.Info.NodeID)
 732                         }
 733                         waitAndAssert(c, defaultReconciliationTimeout, waitActive, checker.True)
 734
 735                         if n.Spec.Role == swarm.NodeRoleManager {
 736                                 c.Assert(n.ManagerStatus, checker.NotNil, check.Commentf("manager status of node %s (manager), reported by %s", n.ID, d.Info.NodeID))
 737                                 if n.ManagerStatus.Leader {
 738                                         leaderFound = true
 739                                 }
 740                                 mCount++
 741                         } else {
 742                                 c.Assert(n.ManagerStatus, checker.IsNil, check.Commentf("manager status of node %s (worker), reported by %s", n.ID, d.Info.NodeID))
 743                                 wCount++
 744                         }
 745                 }
 746                 c.Assert(leaderFound, checker.True, check.Commentf("lack of leader reported by node %s", info.NodeID))
 747                 c.Assert(mCount, checker.Equals, managerCount, check.Commentf("managers count reported by node %s", info.NodeID))
 748                 c.Assert(wCount, checker.Equals, workerCount, check.Commentf("workers count reported by node %s", info.NodeID))
 749         }
 750         c.Assert(totalMCount, checker.Equals, managerCount)
 751         c.Assert(totalWCount, checker.Equals, workerCount)
 752 }
 753
 754 func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) {
 755         mCount, wCount := 5, 1
 756
 757         var nodes []*daemon.Swarm
 758         for i := 0; i < mCount; i++ {
 759                 manager := s.AddDaemon(c, true, true)
 760                 info, err := manager.SwarmInfo()
 761                 c.Assert(err, checker.IsNil)
 762                 c.Assert(info.ControlAvailable, checker.True)
 763                 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
 764                 nodes = append(nodes, manager)
 765         }
 766
 767         for i := 0; i < wCount; i++ {
 768                 worker := s.AddDaemon(c, true, false)
 769                 info, err := worker.SwarmInfo()
 770                 c.Assert(err, checker.IsNil)
 771                 c.Assert(info.ControlAvailable, checker.False)
 772                 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
 773                 nodes = append(nodes, worker)
 774         }
 775
 776         // stop whole cluster
 777         {
 778                 var wg sync.WaitGroup
 779                 wg.Add(len(nodes))
 780                 errs := make(chan error, len(nodes))
 781
 782                 for _, d := range nodes {
 783                         go func(daemon *daemon.Swarm) {
 784                                 defer wg.Done()
 785                                 if err := daemon.StopWithError(); err != nil {
 786                                         errs <- err
 787                                 }
 788                                 // FIXME(vdemeester) This is duplicated…
 789                                 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" {
 790                                         daemon.Root = filepath.Dir(daemon.Root)
 791                                 }
 792                         }(d)
 793                 }
 794                 wg.Wait()
 795                 close(errs)
 796                 for err := range errs {
 797                         c.Assert(err, check.IsNil)
 798                 }
 799         }
 800
 801         // start whole cluster
 802         {
 803                 var wg sync.WaitGroup
 804                 wg.Add(len(nodes))
 805                 errs := make(chan error, len(nodes))
 806
 807                 for _, d := range nodes {
 808                         go func(daemon *daemon.Swarm) {
 809                                 defer wg.Done()
 810                                 if err := daemon.StartWithError("--iptables=false"); err != nil {
 811                                         errs <- err
 812                                 }
 813                         }(d)
 814                 }
 815                 wg.Wait()
 816                 close(errs)
 817                 for err := range errs {
 818                         c.Assert(err, check.IsNil)
 819                 }
 820         }
 821
 822         checkClusterHealth(c, nodes, mCount, wCount)
 823 }
 824
 825 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateWithName(c *check.C) {
 826         d := s.AddDaemon(c, true, true)
 827
 828         instances := 2
 829         id := d.CreateService(c, simpleTestService, setInstances(instances))
 830         waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
 831
 832         service := d.GetService(c, id)
 833         instances = 5
 834
 835         setInstances(instances)(service)
 836         url := fmt.Sprintf("/services/%s/update?version=%d", service.Spec.Name, service.Version.Index)
 837         status, out, err := d.SockRequest("POST", url, service.Spec)
 838         c.Assert(err, checker.IsNil)
 839         c.Assert(status, checker.Equals, http.StatusOK, check.Commentf("output: %q", string(out)))
 840         waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
 841 }
 842
 843 // Unlocking an unlocked swarm results in an error
 844 func (s *DockerSwarmSuite) TestAPISwarmUnlockNotLocked(c *check.C) {
 845         d := s.AddDaemon(c, true, true)
 846         err := d.Unlock(swarm.UnlockRequest{UnlockKey: "wrong-key"})
 847         c.Assert(err, checker.NotNil)
 848         c.Assert(err.Error(), checker.Contains, "swarm is not locked")
 849 }
 850
 851 // #29885
 852 func (s *DockerSwarmSuite) TestAPISwarmErrorHandling(c *check.C) {
 853         ln, err := net.Listen("tcp", fmt.Sprintf(":%d", defaultSwarmPort))
 854         c.Assert(err, checker.IsNil)
 855         defer ln.Close()
 856         d := s.AddDaemon(c, false, false)
 857         err = d.Init(swarm.InitRequest{})
 858         c.Assert(err, checker.NotNil)
 859         c.Assert(err.Error(), checker.Contains, "address already in use")
 860 }
 861
 862 // Test case for 30242, where duplicate networks, with different drivers `bridge` and `overlay`,
 863 // caused both scopes to be `swarm` for `docker network inspect` and `docker network ls`.
 864 // This test makes sure the fixes correctly output scopes instead.
 865 func (s *DockerSwarmSuite) TestAPIDuplicateNetworks(c *check.C) {
 866         d := s.AddDaemon(c, true, true)
 867
 868         name := "foo"
 869         networkCreateRequest := types.NetworkCreateRequest{
 870                 Name: name,
 871                 NetworkCreate: types.NetworkCreate{
 872                         CheckDuplicate: false,
 873                 },
 874         }
 875
 876         var n1 types.NetworkCreateResponse
 877         networkCreateRequest.NetworkCreate.Driver = "bridge"
 878
 879         status, out, err := d.SockRequest("POST", "/networks/create", networkCreateRequest)
 880         c.Assert(err, checker.IsNil, check.Commentf(string(out)))
 881         c.Assert(status, checker.Equals, http.StatusCreated, check.Commentf(string(out)))
 882
 883         c.Assert(json.Unmarshal(out, &n1), checker.IsNil)
 884
 885         var n2 types.NetworkCreateResponse
 886         networkCreateRequest.NetworkCreate.Driver = "overlay"
 887
 888         status, out, err = d.SockRequest("POST", "/networks/create", networkCreateRequest)
 889         c.Assert(err, checker.IsNil, check.Commentf(string(out)))
 890         c.Assert(status, checker.Equals, http.StatusCreated, check.Commentf(string(out)))
 891
 892         c.Assert(json.Unmarshal(out, &n2), checker.IsNil)
 893
 894         var r1 types.NetworkResource
 895
 896         status, out, err = d.SockRequest("GET", "/networks/"+n1.ID, nil)
 897         c.Assert(err, checker.IsNil, check.Commentf(string(out)))
 898         c.Assert(status, checker.Equals, http.StatusOK, check.Commentf(string(out)))
 899
 900         c.Assert(json.Unmarshal(out, &r1), checker.IsNil)
 901
 902         c.Assert(r1.Scope, checker.Equals, "local")
 903
 904         var r2 types.NetworkResource
 905
 906         status, out, err = d.SockRequest("GET", "/networks/"+n2.ID, nil)
 907         c.Assert(err, checker.IsNil, check.Commentf(string(out)))
 908         c.Assert(status, checker.Equals, http.StatusOK, check.Commentf(string(out)))
 909
 910         c.Assert(json.Unmarshal(out, &r2), checker.IsNil)
 911
 912         c.Assert(r2.Scope, checker.Equals, "swarm")
 913 }
 914
 915 // Test case for 30178
 916 func (s *DockerSwarmSuite) TestAPISwarmHealthcheckNone(c *check.C) {
 917         d := s.AddDaemon(c, true, true)
 918
 919         out, err := d.Cmd("network", "create", "-d", "overlay", "lb")
 920         c.Assert(err, checker.IsNil, check.Commentf(out))
 921
 922         instances := 1
 923         d.CreateService(c, simpleTestService, setInstances(instances), func(s *swarm.Service) {
 924                 s.Spec.TaskTemplate.ContainerSpec.Healthcheck = &container.HealthConfig{}
 925                 s.Spec.TaskTemplate.Networks = []swarm.NetworkAttachmentConfig{
 926                         {Target: "lb"},
 927                 }
 928         })
 929
 930         waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
 931
 932         containers := d.ActiveContainers()
 933
 934         out, err = d.Cmd("exec", containers[0], "ping", "-c1", "-W3", "top")
 935         c.Assert(err, checker.IsNil, check.Commentf(out))
 936 }
 937
 938 func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) {
 939         m := s.AddDaemon(c, true, true)
 940         w := s.AddDaemon(c, true, false)
 941
 942         info, err := m.SwarmInfo()
 943         c.Assert(err, checker.IsNil)
 944
 945         currentTrustRoot := info.Cluster.TLSInfo.TrustRoot
 946
 947         // rotate multiple times
 948         for i := 0; i < 4; i++ {
 949                 var cert, key []byte
 950                 if i%2 != 0 {
 951                         cert, _, key, err = initca.New(&csr.CertificateRequest{
 952                                 CN:         "newRoot",
 953                                 KeyRequest: csr.NewBasicKeyRequest(),
 954                                 CA:         &csr.CAConfig{Expiry: ca.RootCAExpiration},
 955                         })
 956                         c.Assert(err, checker.IsNil)
 957                 }
 958                 expectedCert := string(cert)
 959                 m.UpdateSwarm(c, func(s *swarm.Spec) {
 960                         s.CAConfig.SigningCACert = expectedCert
 961                         s.CAConfig.SigningCAKey = string(key)
 962                         s.CAConfig.ForceRotate++
 963                 })
 964
 965                 // poll to make sure update succeeds
 966                 var clusterTLSInfo swarm.TLSInfo
 967                 for j := 0; j < 18; j++ {
 968                         info, err := m.SwarmInfo()
 969                         c.Assert(err, checker.IsNil)
 970
 971                         // the desired CA cert and key is always redacted
 972                         c.Assert(info.Cluster.Spec.CAConfig.SigningCAKey, checker.Equals, "")
 973                         c.Assert(info.Cluster.Spec.CAConfig.SigningCACert, checker.Equals, "")
 974
 975                         clusterTLSInfo = info.Cluster.TLSInfo
 976
 977                         // if root rotation is done and the trust root has changed, we don't have to poll anymore
 978                         if !info.Cluster.RootRotationInProgress && clusterTLSInfo.TrustRoot != currentTrustRoot {
 979                                 break
 980                         }
 981
 982                         // root rotation not done
 983                         time.Sleep(250 * time.Millisecond)
 984                 }
 985                 if cert != nil {
 986                         c.Assert(clusterTLSInfo.TrustRoot, checker.Equals, expectedCert)
 987                 }
 988                 // could take another second or two for the nodes to trust the new roots after the've all gotten
 989                 // new TLS certificates
 990                 for j := 0; j < 18; j++ {
 991                         mInfo := m.GetNode(c, m.NodeID).Description.TLSInfo
 992                         wInfo := m.GetNode(c, w.NodeID).Description.TLSInfo
 993
 994                         if mInfo.TrustRoot == clusterTLSInfo.TrustRoot && wInfo.TrustRoot == clusterTLSInfo.TrustRoot {
 995                                 break
 996                         }
 997
 998                         // nodes don't trust root certs yet
 999                         time.Sleep(250 * time.Millisecond)
1000                 }
1001
1002                 c.Assert(m.GetNode(c, m.NodeID).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo)
1003                 c.Assert(m.GetNode(c, w.NodeID).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo)
1004                 currentTrustRoot = clusterTLSInfo.TrustRoot
1005         }
1006 }
1007
1008 func (s *DockerSwarmSuite) TestAPINetworkInspectWithScope(c *check.C) {
1009         d := s.AddDaemon(c, true, true)
1010
1011         name := "foo"
1012         networkCreateRequest := types.NetworkCreateRequest{
1013                 Name: name,
1014         }
1015
1016         var n types.NetworkCreateResponse
1017         networkCreateRequest.NetworkCreate.Driver = "overlay"
1018
1019         status, out, err := d.SockRequest("POST", "/networks/create", networkCreateRequest)
1020         c.Assert(err, checker.IsNil, check.Commentf(string(out)))
1021         c.Assert(status, checker.Equals, http.StatusCreated, check.Commentf(string(out)))
1022         c.Assert(json.Unmarshal(out, &n), checker.IsNil)
1023
1024         var r types.NetworkResource
1025
1026         status, body, err := d.SockRequest("GET", "/networks/"+name, nil)
1027         c.Assert(err, checker.IsNil, check.Commentf(string(out)))
1028         c.Assert(status, checker.Equals, http.StatusOK, check.Commentf(string(out)))
1029         c.Assert(json.Unmarshal(body, &r), checker.IsNil)
1030         c.Assert(r.Scope, checker.Equals, "swarm")
1031         c.Assert(r.ID, checker.Equals, n.ID)
1032
1033         v := url.Values{}
1034         v.Set("scope", "local")
1035
1036         status, body, err = d.SockRequest("GET", "/networks/"+name+"?"+v.Encode(), nil)
1037         c.Assert(err, checker.IsNil, check.Commentf(string(out)))
1038         c.Assert(status, checker.Equals, http.StatusNotFound, check.Commentf(string(out)))
1039 }