Browse Source
This PR adds supporting features for static membership. It could batch remove consumers from the group with provided group.instance.id list. Reviewers: Guozhang Wang <wangguoz@gmail.com>pull/7317/head
Boyang Chen
5 years ago
committed by
Guozhang Wang
13 changed files with 711 additions and 23 deletions
@ -0,0 +1,49 @@
@@ -0,0 +1,49 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.kafka.clients.admin; |
||||
|
||||
import org.apache.kafka.common.KafkaFuture; |
||||
import org.apache.kafka.common.annotation.InterfaceStability; |
||||
|
||||
import java.util.concurrent.ExecutionException; |
||||
|
||||
/** |
||||
* The result of the {@link KafkaAdminClient#removeMemberFromConsumerGroup(String, RemoveMemberFromConsumerGroupOptions)} call. |
||||
* |
||||
* The API of this class is evolving, see {@link AdminClient} for details. |
||||
*/ |
||||
@InterfaceStability.Evolving |
||||
public class MembershipChangeResult { |
||||
|
||||
private KafkaFuture<RemoveMemberFromGroupResult> future; |
||||
|
||||
MembershipChangeResult(KafkaFuture<RemoveMemberFromGroupResult> future) { |
||||
this.future = future; |
||||
} |
||||
|
||||
/** |
||||
* Return a future which contains the member removal results. |
||||
*/ |
||||
public RemoveMemberFromGroupResult all() throws ExecutionException, InterruptedException { |
||||
return future.get(); |
||||
} |
||||
|
||||
// Visible for testing
|
||||
public KafkaFuture<RemoveMemberFromGroupResult> future() { |
||||
return future; |
||||
} |
||||
} |
||||
@ -0,0 +1,50 @@
@@ -0,0 +1,50 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.kafka.clients.admin; |
||||
|
||||
import org.apache.kafka.common.annotation.InterfaceStability; |
||||
import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; |
||||
import org.apache.kafka.common.requests.JoinGroupRequest; |
||||
|
||||
import java.util.Collection; |
||||
import java.util.List; |
||||
import java.util.stream.Collectors; |
||||
|
||||
/** |
||||
* Options for {@link AdminClient#removeMemberFromConsumerGroup(String, RemoveMemberFromConsumerGroupOptions)}. |
||||
* It carries the members to be removed from the consumer group. |
||||
* |
||||
* The API of this class is evolving, see {@link AdminClient} for details. |
||||
*/ |
||||
@InterfaceStability.Evolving |
||||
public class RemoveMemberFromConsumerGroupOptions extends AbstractOptions<RemoveMemberFromConsumerGroupOptions> { |
||||
|
||||
private List<MemberIdentity> members; |
||||
|
||||
public RemoveMemberFromConsumerGroupOptions(Collection<String> groupInstanceIds) { |
||||
members = groupInstanceIds.stream().map( |
||||
instanceId -> new MemberIdentity() |
||||
.setGroupInstanceId(instanceId) |
||||
.setMemberId(JoinGroupRequest.UNKNOWN_MEMBER_ID) |
||||
).collect(Collectors.toList()); |
||||
} |
||||
|
||||
public List<MemberIdentity> getMembers() { |
||||
return members; |
||||
} |
||||
} |
||||
|
||||
@ -0,0 +1,85 @@
@@ -0,0 +1,85 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.kafka.clients.admin; |
||||
|
||||
import org.apache.kafka.common.KafkaFuture; |
||||
import org.apache.kafka.common.internals.KafkaFutureImpl; |
||||
import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; |
||||
import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; |
||||
import org.apache.kafka.common.protocol.Errors; |
||||
import org.apache.kafka.common.requests.LeaveGroupResponse; |
||||
|
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
/** |
||||
* Result of a batch member removal operation. |
||||
*/ |
||||
public class RemoveMemberFromGroupResult { |
||||
|
||||
private final Errors topLevelError; |
||||
private final Map<MemberIdentity, KafkaFuture<Void>> memberFutures; |
||||
private boolean hasError = false; |
||||
|
||||
RemoveMemberFromGroupResult(LeaveGroupResponse response, |
||||
List<MemberIdentity> membersToRemove) { |
||||
this.topLevelError = response.topLevelError(); |
||||
this.memberFutures = new HashMap<>(membersToRemove.size()); |
||||
|
||||
if (this.topLevelError != Errors.NONE) { |
||||
// If the populated error is a top-level error, fail every member's future.
|
||||
for (MemberIdentity memberIdentity : membersToRemove) { |
||||
KafkaFutureImpl<Void> future = new KafkaFutureImpl<>(); |
||||
future.completeExceptionally(topLevelError.exception()); |
||||
memberFutures.put(memberIdentity, future); |
||||
} |
||||
hasError = true; |
||||
} else { |
||||
for (MemberResponse memberResponse : response.memberResponses()) { |
||||
KafkaFutureImpl<Void> future = new KafkaFutureImpl<>(); |
||||
Errors memberError = Errors.forCode(memberResponse.errorCode()); |
||||
if (memberError != Errors.NONE) { |
||||
future.completeExceptionally(memberError.exception()); |
||||
hasError = true; |
||||
} else { |
||||
future.complete(null); |
||||
} |
||||
memberFutures.put(new MemberIdentity() |
||||
.setMemberId(memberResponse.memberId()) |
||||
.setGroupInstanceId(memberResponse.groupInstanceId()), future); |
||||
} |
||||
} |
||||
} |
||||
|
||||
public Errors topLevelError() { |
||||
return topLevelError; |
||||
} |
||||
|
||||
public boolean hasError() { |
||||
return hasError; |
||||
} |
||||
|
||||
/** |
||||
* Futures of members with corresponding errors when they leave the group. |
||||
* |
||||
* @return list of members who failed to be removed |
||||
*/ |
||||
public Map<MemberIdentity, KafkaFuture<Void>> memberFutures() { |
||||
return memberFutures; |
||||
} |
||||
} |
||||
@ -0,0 +1,50 @@
@@ -0,0 +1,50 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.kafka.clients.admin; |
||||
|
||||
import org.apache.kafka.common.internals.KafkaFutureImpl; |
||||
import org.apache.kafka.common.message.LeaveGroupResponseData; |
||||
import org.apache.kafka.common.requests.LeaveGroupResponse; |
||||
import org.junit.Test; |
||||
|
||||
import java.util.Collections; |
||||
import java.util.concurrent.ExecutionException; |
||||
|
||||
import static org.junit.Assert.assertEquals; |
||||
import static org.junit.Assert.fail; |
||||
|
||||
public class MembershipChangeResultTest { |
||||
|
||||
@Test |
||||
public void testConstructor() { |
||||
KafkaFutureImpl<RemoveMemberFromGroupResult> removeMemberFuture = new KafkaFutureImpl<>(); |
||||
|
||||
MembershipChangeResult changeResult = new MembershipChangeResult(removeMemberFuture); |
||||
assertEquals(removeMemberFuture, changeResult.future()); |
||||
RemoveMemberFromGroupResult removeMemberFromGroupResult = new RemoveMemberFromGroupResult( |
||||
new LeaveGroupResponse(new LeaveGroupResponseData()), |
||||
Collections.emptyList() |
||||
); |
||||
|
||||
removeMemberFuture.complete(removeMemberFromGroupResult); |
||||
try { |
||||
assertEquals(removeMemberFromGroupResult, changeResult.all()); |
||||
} catch (ExecutionException | InterruptedException e) { |
||||
fail("Unexpected exception " + e + " when trying to get remove member result"); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,38 @@
@@ -0,0 +1,38 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.kafka.clients.admin; |
||||
|
||||
import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; |
||||
import org.junit.Test; |
||||
|
||||
import java.util.Collections; |
||||
import java.util.List; |
||||
|
||||
import static org.junit.Assert.assertEquals; |
||||
|
||||
public class RemoveMemberFromConsumerGroupOptionsTest { |
||||
|
||||
@Test |
||||
public void testConstructor() { |
||||
List<String> groupInstanceIds = Collections.singletonList("instance-1"); |
||||
|
||||
RemoveMemberFromConsumerGroupOptions options = new RemoveMemberFromConsumerGroupOptions(groupInstanceIds); |
||||
|
||||
assertEquals(Collections.singletonList( |
||||
new MemberIdentity().setGroupInstanceId("instance-1")), options.getMembers()); |
||||
} |
||||
} |
||||
@ -0,0 +1,154 @@
@@ -0,0 +1,154 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.kafka.clients.admin; |
||||
|
||||
import org.apache.kafka.common.KafkaFuture; |
||||
import org.apache.kafka.common.errors.FencedInstanceIdException; |
||||
import org.apache.kafka.common.errors.GroupAuthorizationException; |
||||
import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; |
||||
import org.apache.kafka.common.message.LeaveGroupResponseData; |
||||
import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; |
||||
import org.apache.kafka.common.protocol.Errors; |
||||
import org.apache.kafka.common.requests.LeaveGroupResponse; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
|
||||
import java.util.Arrays; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.concurrent.ExecutionException; |
||||
|
||||
import static org.junit.Assert.assertFalse; |
||||
import static org.junit.Assert.assertTrue; |
||||
import static org.junit.Assert.assertEquals; |
||||
import static org.junit.Assert.fail; |
||||
|
||||
public class RemoveMemberFromGroupResultTest { |
||||
|
||||
private String instanceOne = "instance-1"; |
||||
private String instanceTwo = "instance-2"; |
||||
private List<MemberIdentity> membersToRemove; |
||||
|
||||
private List<MemberResponse> memberResponses; |
||||
|
||||
@Before |
||||
public void setUp() { |
||||
membersToRemove = Arrays.asList( |
||||
new MemberIdentity() |
||||
.setGroupInstanceId(instanceOne), |
||||
new MemberIdentity() |
||||
.setGroupInstanceId(instanceTwo) |
||||
); |
||||
|
||||
memberResponses = Arrays.asList( |
||||
new MemberResponse() |
||||
.setGroupInstanceId(instanceOne), |
||||
new MemberResponse() |
||||
.setGroupInstanceId(instanceTwo) |
||||
); |
||||
} |
||||
|
||||
@Test |
||||
public void testTopLevelErrorConstructor() { |
||||
RemoveMemberFromGroupResult topLevelErrorResult = |
||||
new RemoveMemberFromGroupResult(new LeaveGroupResponse( |
||||
new LeaveGroupResponseData() |
||||
.setErrorCode(Errors.GROUP_AUTHORIZATION_FAILED.code()) |
||||
.setMembers(memberResponses)), membersToRemove); |
||||
|
||||
assertTrue(topLevelErrorResult.hasError()); |
||||
assertEquals(Errors.GROUP_AUTHORIZATION_FAILED, topLevelErrorResult.topLevelError()); |
||||
|
||||
Map<MemberIdentity, KafkaFuture<Void>> memberFutures = topLevelErrorResult.memberFutures(); |
||||
assertEquals(2, memberFutures.size()); |
||||
for (Map.Entry<MemberIdentity, KafkaFuture<Void>> entry : memberFutures.entrySet()) { |
||||
KafkaFuture<Void> memberFuture = entry.getValue(); |
||||
assertTrue(memberFuture.isCompletedExceptionally()); |
||||
try { |
||||
memberFuture.get(); |
||||
fail("get() should throw ExecutionException"); |
||||
} catch (ExecutionException | InterruptedException e0) { |
||||
assertTrue(e0.getCause() instanceof GroupAuthorizationException); |
||||
} |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void testMemberLevelErrorConstructor() { |
||||
MemberResponse responseOne = new MemberResponse() |
||||
.setGroupInstanceId(instanceOne) |
||||
.setErrorCode(Errors.FENCED_INSTANCE_ID.code()); |
||||
MemberResponse responseTwo = new MemberResponse() |
||||
.setGroupInstanceId(instanceTwo) |
||||
.setErrorCode(Errors.NONE.code()); |
||||
|
||||
RemoveMemberFromGroupResult memberLevelErrorResult = new RemoveMemberFromGroupResult( |
||||
new LeaveGroupResponse(new LeaveGroupResponseData() |
||||
.setMembers(Arrays.asList(responseOne, responseTwo))), |
||||
membersToRemove); |
||||
assertTrue(memberLevelErrorResult.hasError()); |
||||
assertEquals(Errors.NONE, memberLevelErrorResult.topLevelError()); |
||||
|
||||
Map<MemberIdentity, KafkaFuture<Void>> memberFutures = memberLevelErrorResult.memberFutures(); |
||||
assertEquals(2, memberFutures.size()); |
||||
for (Map.Entry<MemberIdentity, KafkaFuture<Void>> entry : memberFutures.entrySet()) { |
||||
KafkaFuture<Void> memberFuture = entry.getValue(); |
||||
if (entry.getKey().groupInstanceId().equals(instanceOne)) { |
||||
assertTrue(memberFuture.isCompletedExceptionally()); |
||||
try { |
||||
memberFuture.get(); |
||||
fail("get() should throw ExecutionException"); |
||||
} catch (ExecutionException | InterruptedException e0) { |
||||
assertTrue(e0.getCause() instanceof FencedInstanceIdException); |
||||
} |
||||
} else { |
||||
assertFalse(memberFuture.isCompletedExceptionally()); |
||||
try { |
||||
memberFuture.get(); |
||||
} catch (ExecutionException | InterruptedException e0) { |
||||
fail("get() shouldn't throw exception"); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void testNoErrorConstructor() { |
||||
MemberResponse responseOne = new MemberResponse() |
||||
.setGroupInstanceId(instanceOne) |
||||
.setErrorCode(Errors.NONE.code()); |
||||
MemberResponse responseTwo = new MemberResponse() |
||||
.setGroupInstanceId(instanceTwo) |
||||
.setErrorCode(Errors.NONE.code()); |
||||
// If no error is specified, failed members are not visible.
|
||||
RemoveMemberFromGroupResult noErrorResult = new RemoveMemberFromGroupResult( |
||||
new LeaveGroupResponse(new LeaveGroupResponseData() |
||||
.setMembers(Arrays.asList(responseOne, responseTwo))), |
||||
membersToRemove); |
||||
assertFalse(noErrorResult.hasError()); |
||||
assertEquals(Errors.NONE, noErrorResult.topLevelError()); |
||||
Map<MemberIdentity, KafkaFuture<Void>> memberFutures = noErrorResult.memberFutures(); |
||||
assertEquals(2, memberFutures.size()); |
||||
for (Map.Entry<MemberIdentity, KafkaFuture<Void>> entry : memberFutures.entrySet()) { |
||||
try { |
||||
entry.getValue().get(); |
||||
} catch (ExecutionException | InterruptedException e0) { |
||||
fail("get() shouldn't throw exception"); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue