Skip to content

feat(inkless): allow diskless switch only when remote storage is enabled [KC-217] #662

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
EelisK wants to merge 1 commit into from
Closed

feat(inkless): allow diskless switch only when remote storage is enabled [KC-217] #662

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions core/src/test/java/kafka/server/InklessConfigsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,8 +391,10 @@ void canEnableDisklessOnClassicTopicWhenAllowFromClassicEnabled() throws Excepti
createTopic(admin, topic, Map.of());
assertEquals("false", getTopicConfig(admin, topic).get(DISKLESS_ENABLE_CONFIG));

alterTopicConfig(admin, topic, Map.of(DISKLESS_ENABLE_CONFIG, "true"));
assertEquals("true", getTopicConfig(admin, topic).get(DISKLESS_ENABLE_CONFIG));
alterTopicConfig(admin, topic, Map.of(REMOTE_LOG_STORAGE_ENABLE_CONFIG, "true", DISKLESS_ENABLE_CONFIG, "true"));
Map<String, String> topicConfig = getTopicConfig(admin, topic);
assertEquals("true", topicConfig.get(DISKLESS_ENABLE_CONFIG));
assertEquals("true", topicConfig.get(REMOTE_LOG_STORAGE_ENABLE_CONFIG));
} finally {
cluster.close();
}
Expand Down
41 changes: 36 additions & 5 deletions core/src/test/java/kafka/server/InklessTopicTypeSwitcherClusterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,9 @@ public void switchClassicTopicToDiskless(final AlterConfigsMode alterConfigsMode

try {
log.warn("[stage=switch-start] Enabling diskless for topic={} via {}", classicToDisklessTopic, alterConfigsMode);
alterTopicConfig(admin, classicToDisklessTopic, Map.of(TopicConfig.DISKLESS_ENABLE_CONFIG, "true"), alterConfigsMode);
alterTopicConfig(admin, classicToDisklessTopic, Map.of(
TopicConfig.DISKLESS_ENABLE_CONFIG, "true",
TopicConfig.REMOTE_LOG_STORAGE_ENABLE_CONFIG, "true"), alterConfigsMode);

log.warn("[stage=await-switch] Waiting for diskless.enable=true on topic={}", classicToDisklessTopic);
waitForTopicDisklessValue(admin, classicToDisklessTopic, "true");
Expand Down Expand Up @@ -404,6 +406,27 @@ private void alterTopicConfigWithIncrementalAlterConfigs(final Admin admin,
admin.incrementalAlterConfigs(Map.of(topicResource, operations)).all().get(20, TimeUnit.SECONDS);
}

@Test
public void testSwitchRejectedWhenRemoteStorageDisabled() throws Exception {
final String topic = "switch-remote-disabled-" + UUID.randomUUID().toString().substring(0, 8);

try (Admin admin = AdminClient.create(baseClientConfigs())) {
// Create a classic topic with RF=3
admin.createTopics(List.of(
new NewTopic(topic, 1, (short) 3)
.configs(Map.of(TopicConfig.DISKLESS_ENABLE_CONFIG, "false"))
)).all().get(30, TimeUnit.SECONDS);

// Attempt to switch to diskless
final ExecutionException ex = assertThrows(ExecutionException.class, () ->
alterTopicConfigWithIncrementalAlterConfigs(admin, topic, Map.of(
TopicConfig.DISKLESS_ENABLE_CONFIG, "true")));
assertInstanceOf(InvalidConfigurationException.class, ex.getCause());
assertTrue(ex.getCause().getMessage().contains("remote storage must be enabled"),
"Expected 'remote storage must be enabled' in: " + ex.getCause().getMessage());
}
}

@Test
public void testSwitchRejectedWhenPartitionIsOfflineOrUnderReplicated() throws Exception {
final String topic = "switch-unhealthy-" + UUID.randomUUID().toString().substring(0, 8);
Expand All @@ -422,7 +445,9 @@ public void testSwitchRejectedWhenPartitionIsOfflineOrUnderReplicated() throws E

// Attempt to switch to diskless (should fail with INVALID_CONFIG due to under-replication)
final ExecutionException ex = assertThrows(ExecutionException.class, () ->
alterTopicConfigWithIncrementalAlterConfigs(admin, topic, Map.of(TopicConfig.DISKLESS_ENABLE_CONFIG, "true")));
alterTopicConfigWithIncrementalAlterConfigs(admin, topic, Map.of(
TopicConfig.DISKLESS_ENABLE_CONFIG, "true",
TopicConfig.REMOTE_LOG_STORAGE_ENABLE_CONFIG, "true")));
assertInstanceOf(InvalidConfigurationException.class, ex.getCause());
assertTrue(ex.getCause().getMessage().contains("under-replicated"),
"Expected 'under-replicated' in: " + ex.getCause().getMessage());
Expand All @@ -432,8 +457,12 @@ public void testSwitchRejectedWhenPartitionIsOfflineOrUnderReplicated() throws E
waitForIsrRecovery(admin, topic, 0, 3);

// Now the switch should succeed
alterTopicConfigWithIncrementalAlterConfigs(admin, topic, Map.of(TopicConfig.DISKLESS_ENABLE_CONFIG, "true"));
assertEquals("true", getTopicConfig(admin, topic).get(TopicConfig.DISKLESS_ENABLE_CONFIG));
alterTopicConfigWithIncrementalAlterConfigs(admin, topic, Map.of(
TopicConfig.DISKLESS_ENABLE_CONFIG, "true",
TopicConfig.REMOTE_LOG_STORAGE_ENABLE_CONFIG, "true"));
Map<String, String> topicConfig = getTopicConfig(admin, topic);
assertEquals("true", topicConfig.get(TopicConfig.DISKLESS_ENABLE_CONFIG));
assertEquals("true", topicConfig.get(TopicConfig.REMOTE_LOG_STORAGE_ENABLE_CONFIG));
}
}

Expand Down Expand Up @@ -486,7 +515,9 @@ public void testUncleanLeaderElectionRejectedWhileSwitchPending() throws Excepti

// Enable diskless to mark partition as switch pending
alterTopicConfigWithIncrementalAlterConfigs(
admin, topic, Map.of(TopicConfig.DISKLESS_ENABLE_CONFIG, "true"));
admin, topic, Map.of(
TopicConfig.DISKLESS_ENABLE_CONFIG, "true",
TopicConfig.REMOTE_LOG_STORAGE_ENABLE_CONFIG, "true"));

// Should fail trying to enable unclean leader election
final ExecutionException ex = assertThrows(ExecutionException.class, () ->
Expand Down
7 changes: 4 additions & 3 deletions core/src/test/scala/unit/kafka/log/LogConfigTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -677,7 +677,8 @@ class LogConfigTest {
// Case 7: diskless is enabled and remote storage becomes enabled
assertValid(existingWithDisklessTrueRemoteFalse, setDisklessTrueRemoteStorageTrue, kafkaConfig, disklessAllowFromClassic = true, remoteStorageConsolidationEnabled = true)
assertValid(existingWithDisklessTrueRemoteFalse, setDisklessTrueRemoteStorageTrue, kafkaConfig, remoteStorageConsolidationEnabled = true)
assertInvalid(existingWithDisklessTrueRemoteFalse, setDisklessTrueRemoteStorageTrue, mutualExclusionError, kafkaConfig, disklessAllowFromClassic = true)
// Enabling remote storage on a diskless topic is a valid classic-to-diskless switch
assertValid(existingWithDisklessTrueRemoteFalse, setDisklessTrueRemoteStorageTrue, kafkaConfig, disklessAllowFromClassic = true)
assertInvalid(existingWithDisklessTrueRemoteFalse, setDisklessTrueRemoteStorageTrue, mutualExclusionError, kafkaConfig)

// Case 8: if diskless and remote is enabled, can't disable remote storage
Expand Down Expand Up @@ -718,8 +719,8 @@ class LogConfigTest {
assertValid(existingWithRemoteTrue, setDisklessTrueWithExistingRemoteTrue, kafkaConfig, disklessAllowFromClassic = true, remoteStorageConsolidationEnabled = true)
// CLASSIC→DISKLESS direct switch: both diskless.enable=true and remote.storage.enable=true on a topic with neither config
assertValid(existingWithoutDisklessOrRemote, setDisklessTrueWithExistingRemoteTrue, kafkaConfig, disklessAllowFromClassic = true, remoteStorageConsolidationEnabled = true)
// Same switch rejected without consolidation gate
assertInvalid(existingWithoutDisklessOrRemote, setDisklessTrueWithExistingRemoteTrue, mutualExclusionError, kafkaConfig, disklessAllowFromClassic = true, remoteStorageConsolidationEnabled = false)
// Same switch is allowed without consolidation: the switch only requires allow-from-classic
assertValid(existingWithoutDisklessOrRemote, setDisklessTrueWithExistingRemoteTrue, kafkaConfig, disklessAllowFromClassic = true, remoteStorageConsolidationEnabled = false)

// Case 2: set diskless.enable=false with allowFromClassic=true - disabling diskless is still forbidden
val setDisklessFalse = topicProps(TopicConfig.DISKLESS_ENABLE_CONFIG -> "false")
Expand Down
10 changes: 9 additions & 1 deletion metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3090,8 +3090,16 @@ private ApiError validateDisklessSwitchInvariants(
}
}

// All partitions must be healthy to initiate a switch
if (initiatingSwitch) {
ConfigEntry remoteStorageEntry = effectiveTopicConfigs.get(REMOTE_LOG_STORAGE_ENABLE_CONFIG);
boolean remoteStorageEnabled = remoteStorageEntry != null
&& Boolean.parseBoolean(remoteStorageEntry.value());
if (!remoteStorageEnabled && !isDisklessRemoteStorageConsolidationEnabled) {
return new ApiError(INVALID_CONFIG,
"Cannot switch topic " + topicName + " to diskless: " +
"remote storage must be enabled.");
}
// All partitions must be healthy to initiate a switch
return validatePartitionsForSwitch(topicInfo);
}

Expand Down
Loading
Loading