Run Member tests in parallel

server/embed/config.go

@@ -598,6 +598,14 @@ func NewConfig() *Config {
 }
 
 func (cfg *Config) AddFlags(fs *flag.FlagSet) {
+	cfg.AddFlagsWithoutGlobals(fs)
+
+	// raft connection timeouts
+	fs.DurationVar(&rafthttp.ConnReadTimeout, "raft-read-timeout", rafthttp.DefaultConnReadTimeout, "Read timeout set on each rafthttp connection")
+	fs.DurationVar(&rafthttp.ConnWriteTimeout, "raft-write-timeout", rafthttp.DefaultConnWriteTimeout, "Write timeout set on each rafthttp connection")
+}
+
+func (cfg *Config) AddFlagsWithoutGlobals(fs *flag.FlagSet) {
 	// member
 	fs.StringVar(&cfg.Dir, "data-dir", cfg.Dir, "Path to the data directory.")
 	fs.StringVar(&cfg.WalDir, "wal-dir", cfg.WalDir, "Path to the dedicated wal directory.")
@@ -640,10 +648,6 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) {
 
 	fs.Var(flags.NewUint32Value(cfg.MaxConcurrentStreams), "max-concurrent-streams", "Maximum concurrent streams that each client can open at a time.")
 
-	// raft connection timeouts
-	fs.DurationVar(&rafthttp.ConnReadTimeout, "raft-read-timeout", rafthttp.DefaultConnReadTimeout, "Read timeout set on each rafthttp connection")
-	fs.DurationVar(&rafthttp.ConnWriteTimeout, "raft-write-timeout", rafthttp.DefaultConnWriteTimeout, "Write timeout set on each rafthttp connection")
-
 	// clustering
 	fs.Var(
 		flags.NewUniqueURLsWithExceptions(DefaultInitialAdvertisePeerURLs, ""),

tests/common/member_test.go

@@ -34,6 +34,8 @@ func TestMemberList(t *testing.T) {
 
 	for _, tc := range clusterTestCases() {
 		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
 			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 			defer cancel()
 			clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config))
@@ -113,6 +115,8 @@ func TestMemberAdd(t *testing.T) {
 		for _, quorumTc := range quorumTcs {
 			for _, clusterTc := range clusterTestCases() {
 				t.Run(learnerTc.name+"/"+quorumTc.name+"/"+clusterTc.name, func(t *testing.T) {
+					t.Parallel()
+
 					ctxTimeout := 10 * time.Second
 					if quorumTc.waitForQuorum {
 						ctxTimeout += etcdserver.HealthInterval
@@ -198,6 +202,8 @@ func TestMemberRemove(t *testing.T) {
 				continue
 			}
 			t.Run(quorumTc.name+"/"+clusterTc.name, func(t *testing.T) {
+				t.Parallel()
+
 				ctx, cancel := context.WithTimeout(context.Background(), 14*time.Second)
 				defer cancel()
 				c := clusterTc.config

tests/framework/e2e/cluster.go

@@ -407,9 +407,6 @@ func InitEtcdProcessCluster(t testing.TB, cfg *EtcdProcessClusterConfig) (*EtcdP
 	if cfg.Logger == nil {
 		cfg.Logger = zaptest.NewLogger(t)
 	}
-	if cfg.BasePort == 0 {
-		cfg.BasePort = EtcdProcessBasePort
-	}
 	if cfg.ServerConfig.SnapshotCount == 0 {
 		cfg.ServerConfig.SnapshotCount = etcdserver.DefaultSnapshotCount
 	}
@@ -518,6 +515,16 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
 	peer2Port := port + 3
 	clientHTTPPort := port + 4
 
+	var allocatedPorts []int
+	if cfg.BasePort == 0 {
+		clientPort = uniquePorts.Alloc()
+		peerPort = uniquePorts.Alloc()
+		metricsPort = uniquePorts.Alloc()
+		peer2Port = uniquePorts.Alloc()
+		clientHTTPPort = uniquePorts.Alloc()
+		allocatedPorts = []int{clientPort, peerPort, metricsPort, peer2Port, clientHTTPPort}
+	}
+
 	if cfg.Client.ConnectionType == ClientTLSAndNonTLS {
 		curl = clientURL(cfg.ClientScheme(), clientPort, ClientNonTLS)
 		curls = []string{curl, clientURL(cfg.ClientScheme(), clientPort, ClientTLS)}
@@ -639,7 +646,8 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
 	}
 	var gofailPort int
 	if cfg.GoFailEnabled {
-		gofailPort = (i+1)*10000 + 2381
+		gofailPort = uniquePorts.Alloc()
+		allocatedPorts = append(allocatedPorts, gofailPort)
 		envVars["GOFAIL_HTTP"] = fmt.Sprintf("127.0.0.1:%d", gofailPort)
 	}
 
@@ -662,12 +670,13 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
 		GoFailClientTimeout: cfg.GoFailClientTimeout,
 		Proxy:               proxyCfg,
 		LazyFSEnabled:       cfg.LazyFSEnabled,
+		AllocatedPorts:      allocatedPorts,
 	}
 }
 
 func values(cfg embed.Config) map[string]string {
 	fs := flag.NewFlagSet("etcd", flag.ContinueOnError)
-	cfg.AddFlags(fs)
+	cfg.AddFlagsWithoutGlobals(fs)
 	values := map[string]string{}
 	fs.VisitAll(func(f *flag.Flag) {
 		value := f.Value.String()

tests/framework/e2e/cluster_test.go

@@ -34,12 +34,12 @@ func TestEtcdServerProcessConfig(t *testing.T) {
 	}{
 		{
 			name:   "Default",
-			config: NewConfig(),
+			config: NewConfig(WithBasePort(20000)),
 			expectArgsContain: []string{
-				"--listen-client-urls=http://localhost:0",
-				"--advertise-client-urls=http://localhost:0",
-				"--listen-peer-urls=http://localhost:1",
-				"--initial-advertise-peer-urls=http://localhost:1",
+				"--listen-client-urls=http://localhost:20000",
+				"--advertise-client-urls=http://localhost:20000",
+				"--listen-peer-urls=http://localhost:20001",
+				"--initial-advertise-peer-urls=http://localhost:20001",
 				"--initial-cluster-token=new",
 				"--snapshot-count=10000",
 			},

tests/framework/e2e/etcd_process.go

@@ -88,10 +88,11 @@ type EtcdServerProcessConfig struct {
 
 	Name string
 
-	PeerURL       url.URL
-	ClientURL     string
-	ClientHTTPURL string
-	MetricsURL    string
+	PeerURL        url.URL
+	ClientURL      string
+	ClientHTTPURL  string
+	MetricsURL     string
+	AllocatedPorts []int
 
 	InitialToken        string
 	InitialCluster      string
@@ -248,6 +249,11 @@ func (ep *EtcdServerProcess) Close() error {
 		ep.cfg.lg.Info("removing directory", zap.String("data-dir", ep.cfg.DataDirPath))
 		return os.RemoveAll(ep.cfg.DataDirPath)
 	}
+
+	for _, port := range ep.cfg.AllocatedPorts {
+		uniquePorts.Free(port)
+	}
+	ep.cfg.AllocatedPorts = nil
 	return nil
 }
 

tests/framework/e2e/port_alloc.go

@@ -0,0 +1,58 @@
+// Copyright 2024 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import "sync"
+
+// uniquePorts is a global instance of testPorts.
+var uniquePorts *testPorts
+
+func init() {
+	uniquePorts = newTestPorts(11000, 19000)
+}
+
+// testPorts is used to allocate listen ports for etcd instance in tests
+// in a safe way for concurrent use (i.e. running tests in parallel).
+type testPorts struct {
+	mux    sync.Mutex
+	unused map[int]bool
+}
+
+// newTestPorts keeps track of unused ports in the specified range.
+func newTestPorts(start, end int) *testPorts {
+	m := make(map[int]bool, end-start)
+	for i := start; i < end; i++ {
+		m[i] = true
+	}
+	return &testPorts{unused: m}
+}
+
+// Alloc allocates a new port or panics if none is available.
+func (pa *testPorts) Alloc() int {
+	pa.mux.Lock()
+	defer pa.mux.Unlock()
+	for port := range pa.unused {
+		delete(pa.unused, port)
+		return port
+	}
+	panic("all ports are used")
+}
+
+// Free makes port available for allocation through Alloc.
+func (pa *testPorts) Free(port int) {
+	pa.mux.Lock()
+	defer pa.mux.Unlock()
+	pa.unused[port] = true
+}

tests/framework/integration/cluster.go

@@ -260,13 +260,12 @@ func (c *Cluster) ProtoMembers() []*pb.Member {
 }
 
 func (c *Cluster) mustNewMember(t testutil.TB) *Member {
-	memberNumber := c.LastMemberNum
+	uniqueNumber := atomic.AddInt32(&UniqueNumber, 1)*10 + int32(c.LastMemberNum)
 	c.LastMemberNum++
 
 	m := MustNewMember(t,
 		MemberConfig{
-			Name:                        fmt.Sprintf("m%v", memberNumber),
-			MemberNumber:                memberNumber,
+			Name:                        fmt.Sprintf("m%v", uniqueNumber),
 			AuthToken:                   c.Cfg.AuthToken,
 			PeerTLS:                     c.Cfg.PeerTLS,
 			ClientTLS:                   c.Cfg.ClientTLS,
@@ -549,7 +548,6 @@ func NewListenerWithAddr(t testutil.TB, addr string) net.Listener {
 type Member struct {
 	config.ServerConfig
 	UniqNumber                     int
-	MemberNumber                   int
 	Port                           string
 	PeerListeners, ClientListeners []net.Listener
 	GRPCListener                   net.Listener
@@ -591,7 +589,6 @@ type Member struct {
 type MemberConfig struct {
 	Name                        string
 	UniqNumber                  int64
-	MemberNumber                int
 	PeerTLS                     *transport.TLSInfo
 	ClientTLS                   *transport.TLSInfo
 	AuthToken                   string
@@ -624,8 +621,7 @@ type MemberConfig struct {
 func MustNewMember(t testutil.TB, mcfg MemberConfig) *Member {
 	var err error
 	m := &Member{
-		MemberNumber: mcfg.MemberNumber,
-		UniqNumber:   int(atomic.AddInt32(&UniqueCount, 1)),
+		UniqNumber: int(atomic.AddInt32(&UniqueCount, 1)),
 	}
 
 	peerScheme := SchemeFromTLSInfo(mcfg.PeerTLS)