Skip to content

Commit c0bf25b

Browse files
mworzalamworzala
committed
feat: multi-version backend support in proxy by passing around the stored protocol version
1 parent 27f5d50 commit c0bf25b

29 files changed

Lines changed: 666 additions & 173 deletions

File tree

‎bin/api-server/src/main/java/net/hollowcube/apiserver/session/SessionServiceImpl.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,10 @@ public int onlinePlayers() {
2525
var hub = db.servers.findHub(exclude);
2626
return hub == null ? null : new GameServer(hub.id(), hub.clusterIp(), hub.protocolVersion());
2727
}
28+
29+
@Override
30+
public @Nullable GameServer findServer(String id) {
31+
var server = db.servers.findServer(id);
32+
return server == null ? null : new GameServer(server.id(), server.clusterIp(), server.protocolVersion());
33+
}
2834
}

‎bin/api-server/src/test/java/net/hollowcube/apiserver/session/SessionServiceImplTest.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,4 +88,18 @@ insert into server_states (id, role, status, cluster_ip) values
8888
assertNull(sessions.findHub("hub-a"));
8989
assertEquals(new GameServer("hub-a", "10.0.0.3", 0), sessions.findHub("hub-b"));
9090
}
91+
92+
@Test
93+
void findServer_isNullOnceThePodIsGone() {
94+
TEST_DB.seed(
95+
"""
96+
insert into server_states (id, role, status, cluster_ip, protocol_version) values
97+
('isolate-a', 'map-isolate', 0, '10.0.0.4', 777),
98+
('isolate-b', 'map-isolate', 0, '', 0)"""
99+
);
100+
101+
assertEquals(new GameServer("isolate-a", "10.0.0.4", 777), sessions.findServer("isolate-a"));
102+
assertNull(sessions.findServer("isolate-b"));
103+
assertNull(sessions.findServer("isolate-c"));
104+
}
91105
}

‎bin/development/src/main/java/net/hollowcube/mapmaker/dev/DevServer.java‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import net.hollowcube.ipc.player.PlayerServer;
3232
import net.hollowcube.ipc.player.SocialServer;
3333
import net.hollowcube.ipc.replay.ReplayServer;
34+
import net.hollowcube.ipc.session.GameServer;
3435
import net.hollowcube.ipc.session.SessionServer;
3536
import net.hollowcube.mapmaker.config.ConfigLoaderV3;
3637
import net.hollowcube.mapmaker.dev.commands.AcDevCommand;
@@ -64,6 +65,7 @@
6465
import net.minestom.server.event.trait.PlayerEvent;
6566
import net.minestom.server.ping.Status;
6667
import org.jetbrains.annotations.NotNull;
68+
import org.jetbrains.annotations.Nullable;
6769

6870
import java.net.http.HttpClient;
6971
import java.nio.file.Path;
@@ -174,7 +176,8 @@ private void startApiWorker() {
174176
new ChatServiceImpl(db, nats),
175177
new ReplayServiceImpl(db, s3),
176178
new MapServiceImpl(db, mapWorlds, nats, redis, PostHog.getClient(), Duration.ZERO),
177-
new PlayerServiceImpl(db), new SocialServiceImpl(db, notifications), notifications);
179+
new PlayerServiceImpl(db), new SocialServiceImpl(db, notifications), notifications,
180+
new DevSessionService(new SessionServiceImpl(db)));
178181
return ipc;
179182
}
180183

@@ -187,7 +190,7 @@ public void registerHttpRoutes(@NotNull HttpServerWrapper http) {
187190
MAX_TRACE_BYTES);
188191
http.addRoute(HeadDatabaseServer.PATH, new HeadDatabaseServer(ipc.headDatabase()));
189192
http.addRoute(ChatServer.PATH, new ChatServer(ipc.chat()));
190-
http.addRoute(SessionServer.PATH, new SessionServer(new SessionServiceImpl(db)));
193+
http.addRoute(SessionServer.PATH, new SessionServer(ipc.sessions()));
191194
http.addRoute(AnticheatServer.PATH, new AnticheatServer(new AnticheatServiceImpl(db, store)));
192195
http.addRoute(MapServer.PATH, new MapServer(ipc.maps()));
193196
http.addRoute(ReplayServer.PATH, new ReplayServer(ipc.replays()));
@@ -313,4 +316,23 @@ protected void handleServerListPing(@NotNull ServerListPingEvent event) {
313316
.build());
314317
}
315318

319+
320+
/// The dev server is its own hub, and nothing tracks it in `server_states` as one, so a proxy in
321+
/// front of it (`runProxy`) is always handed this server.
322+
private record DevSessionService(SessionServiceImpl sessions) implements net.hollowcube.ipc.session.SessionService {
323+
@Override
324+
public int onlinePlayers() {
325+
return sessions.onlinePlayers();
326+
}
327+
328+
@Override
329+
public GameServer findHub(@Nullable String exclude) {
330+
return new GameServer("devserver", "127.0.0.1", MinecraftServer.PROTOCOL_VERSION);
331+
}
332+
333+
@Override
334+
public @Nullable GameServer findServer(String id) {
335+
return "devserver".equals(id) ? findHub(null) : null;
336+
}
337+
}
316338
}

‎bin/hub/src/main/java/net/hollowcube/mapmaker/hub/HubServer.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@
2626
import net.hollowcube.mapmaker.map.setting.TimeOfDay;
2727
import net.hollowcube.mapmaker.misc.ProxySupport;
2828
import net.hollowcube.mapmaker.misc.ResourcePackManager;
29-
import net.hollowcube.mapmaker.player.JoinHubRequest;
30-
import net.hollowcube.mapmaker.player.SessionService;
3129
import net.hollowcube.mapmaker.session.Presence;
3230
import net.hollowcube.mapmaker.util.AbstractHttpService;
3331
import net.hollowcube.mapmaker.util.ServerBeginShutdownEvent;
@@ -205,13 +203,14 @@ private int transferToAnotherHub(List<Player> players) {
205203
var stranded = 0;
206204
for (var player : players) {
207205
try {
208-
var hub = sessionService().joinHubV2(new JoinHubRequest(
209-
player.getUuid().toString(), AbstractHttpService.hostname));
206+
var hub = api().sessions.findHub(AbstractHttpService.hostname);
207+
if (hub == null) {
208+
stranded++;
209+
continue;
210+
}
210211

211212
var state = new HubTransferData(player.getPosition(), player.getHeldSlot());
212-
ProxySupport.transferWithData(player, hub.serverClusterIp(), state);
213-
} catch (SessionService.NoAvailableServerException ignored) {
214-
stranded++;
213+
ProxySupport.transferWithData(player, hub, state);
215214
} catch (Exception e) {
216215
ExceptionReporter.reportException(e, player);
217216
}

‎bin/hub/src/main/java/net/hollowcube/mapmaker/hub/HubServerBridge.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ public void joinMap(Player player, JoinConfig joinConfig) {
5353
case SPECTATING -> MapPresence.STATE_SPECTATING;
5454
}, joinConfig.source(), joinConfig.isolateOverride()));
5555
logger.info("join map result: {}", res);
56-
ProxySupport.transfer(player, res.serverClusterIp());
56+
ProxySupport.transfer(player, res);
5757
} catch (Exception e) {
5858
if (!(e instanceof SessionService.NoAvailableServerException))
5959
ExceptionReporter.reportException(e, player);

‎bin/map-isolate/src/main/java/net/hollowcube/mapmaker/isolate/MapIsolateBridge.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
import net.hollowcube.mapmaker.api.ApiClient;
88
import net.hollowcube.mapmaker.map.runtime.ServerBridge;
99
import net.hollowcube.mapmaker.misc.ProxySupport;
10-
import net.hollowcube.mapmaker.player.JoinHubRequest;
1110
import net.hollowcube.mapmaker.player.JoinMapRequest;
1211
import net.hollowcube.mapmaker.player.SessionService;
1312
import net.hollowcube.mapmaker.session.MapPresence;
@@ -19,7 +18,6 @@
1918
import java.util.concurrent.CompletableFuture;
2019

2120
import static net.hollowcube.common.util.PlayerUtil.onConfigOrDisconnect;
22-
import static net.hollowcube.mapmaker.player.LocalPlayer.localPlayer;
2321

2422
public class MapIsolateBridge implements ServerBridge {
2523
private static final Logger logger = LoggerFactory.getLogger(MapIsolateBridge.class);
@@ -59,7 +57,7 @@ public void joinMap(Player player, JoinConfig joinConfig) {
5957
var response = sessionService.joinMapV2(new JoinMapRequest(playerId, joinConfig.mapId(), targetState, joinConfig.source(), joinConfig.isolateOverride()));
6058
logger.info("join map result: {}", response);
6159

62-
ProxySupport.transfer(player, response.serverClusterIp());
60+
ProxySupport.transfer(player, response);
6361
} catch (Exception e) {
6462
ExceptionReporter.reportException(e, player);
6563
player.sendMessage(Component.text("An error occurred while trying to join the map. Please try again later."));
@@ -69,13 +67,17 @@ public void joinMap(Player player, JoinConfig joinConfig) {
6967
@Override
7068
public void joinHub(Player player) {
7169
try {
70+
var hub = api.sessions.findHub(null);
71+
if (hub == null) {
72+
player.sendMessage("No hub server is available!");
73+
return;
74+
}
75+
7276
var future = new CompletableFuture<Void>();
7377
onConfigOrDisconnect(player, () -> future.complete(null));
7478

75-
var playerData = localPlayer(player);
76-
var res = sessionService.joinHubV2(new JoinHubRequest(playerData.id().toString()));
77-
logger.info("join hub result: {}", res);
78-
ProxySupport.transfer(player, res.serverClusterIp());
79+
logger.info("join hub result: {}", hub);
80+
ProxySupport.transfer(player, hub);
7981

8082
FutureUtil.getUnchecked(future); // Wait until not in instance
8183
} catch (Exception e) {

‎bin/map/src/main/java/net/hollowcube/mapmaker/map/MapServerBridge.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,7 @@
66
import net.hollowcube.mapmaker.ExceptionReporter;
77
import net.hollowcube.mapmaker.map.runtime.ServerBridge;
88
import net.hollowcube.mapmaker.misc.ProxySupport;
9-
import net.hollowcube.mapmaker.player.JoinHubRequest;
109
import net.hollowcube.mapmaker.player.JoinMapRequest;
11-
import net.hollowcube.mapmaker.player.SessionService;
1210
import net.hollowcube.mapmaker.session.MapPresence;
1311
import net.hollowcube.mapmaker.util.AbstractHttpService;
1412
import net.kyori.adventure.text.Component;
@@ -67,7 +65,7 @@ public void joinMap(@NotNull Player player, @NotNull JoinConfig joinConfig) {
6765
this.moveBetweenMapsOnThisServer(player, joinConfig.mapId(), targetState);
6866
} else {
6967
logger.info("moving to other server");
70-
ProxySupport.transfer(player, response.serverClusterIp());
68+
ProxySupport.transfer(player, response);
7169
}
7270
} catch (Exception e) {
7371
ExceptionReporter.reportException(e, player);
@@ -78,17 +76,19 @@ public void joinMap(@NotNull Player player, @NotNull JoinConfig joinConfig) {
7876
@Override
7977
public void joinHub(@NotNull Player player) {
8078
try {
79+
var hub = server.api().sessions.findHub(null);
80+
if (hub == null) {
81+
player.sendMessage("No hub server is available!");
82+
return;
83+
}
84+
8185
var future = new CompletableFuture<Void>();
8286
onConfigOrDisconnect(player, () -> future.complete(null));
8387

84-
var playerData = localPlayer(player);
85-
var res = server.sessionService().joinHubV2(new JoinHubRequest(playerData.id().toString()));
86-
logger.info("join hub result: {}", res);
87-
ProxySupport.transfer(player, res.serverClusterIp());
88+
logger.info("join hub result: {}", hub);
89+
ProxySupport.transfer(player, hub);
8890

8991
FutureUtil.getUnchecked(future); // Wait until not in instance
90-
} catch (SessionService.NoAvailableServerException ignored) {
91-
player.sendMessage("No hub server is available!");
9292
} catch (Exception e) {
9393
ExceptionReporter.reportException(e, player);
9494
player.sendMessage(Component.text("An error occurred while trying to return to the hub. Please try again later."));

‎bin/proxy-plugin/build.gradle.kts‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ dependencies {
2121
// VelocityInternals reaches ConnectedPlayer#getConnection()#getChannel() through MethodHandles.
2222
annotationProcessor(libs.velocity.api)
2323
compileOnly(libs.velocity.api)
24+
// The api of the very jar the proxy image ships, so what compiles is what runs.
25+
compileOnly(files("proxy/plugins/ViaVersion-5.12.0.jar"))
2426

2527
// The session count on the server list ping and the trace store the shipper puts to are ipc
2628
// services, so the generated SessionClient and AnticheatClient ride along. They drag in
@@ -42,6 +44,9 @@ dependencies {
4244
implementation(libs.prometheus)
4345
implementation(libs.prometheus.httpserver)
4446

47+
// BackendsTest stands a fake ProxyServer up in front of Backends.
48+
testImplementation(libs.velocity.api)
49+
4550
// Tap tests drive an EmbeddedChannel; velocity-api brings no netty of its own.
4651
testImplementation(libs.netty.transport)
4752
testImplementation(libs.netty.codec)
@@ -163,11 +168,12 @@ val stageProxy = tasks.register<Sync>("stageProxy") {
163168
from(tasks.shadowJar) { into("plugins") }
164169
}
165170

166-
// A local run of the very thing that ships, differing only in what has to differ: the hub to
167-
// forward to, the bind port, and the forwarding secret, each of which is a gradle property.
171+
// A local run of the very thing that ships, differing only in what has to differ: the bind port
172+
// and the forwarding secret, each of which is a gradle property. The hub comes from the ipc
173+
// session service like it does in production; DevServer answers with itself.
168174
// Velocity's own state (logs, lang, plugin data) is left alone between runs.
169175
//
170-
// ./gradlew :bin:proxy-plugin:runProxy -PproxyHub=127.0.0.1:25565 -PproxySecret=abcdef
176+
// ./gradlew :bin:proxy-plugin:runProxy -PproxySecret=abcdef
171177
val proxyRunDir = layout.buildDirectory.dir("proxy/run")
172178
val prepareProxyRun = tasks.register<Sync>("prepareProxyRun") {
173179
group = "proxy"
@@ -179,11 +185,9 @@ val prepareProxyRun = tasks.register<Sync>("prepareProxyRun") {
179185
include("plugins/*/**")
180186
}
181187

182-
val hub = providers.gradleProperty("proxyHub").getOrElse("127.0.0.1:25565")
183188
val bind = providers.gradleProperty("proxyBind").getOrElse("0.0.0.0:25577")
184189
val secret = providers.gradleProperty("proxySecret").getOrElse("abcdef")
185190
val cookieSecret = providers.gradleProperty("proxyCookieSecret").getOrElse("local dev")
186-
inputs.property("hub", hub)
187191
inputs.property("bind", bind)
188192
inputs.property("secret", secret)
189193
inputs.property("cookieSecret", cookieSecret)
@@ -192,16 +196,15 @@ val prepareProxyRun = tasks.register<Sync>("prepareProxyRun") {
192196
val dir = runDir.get().asFile
193197
val config = dir.resolve("velocity.toml")
194198
config.writeText(config.readText()
195-
.replace(Regex("(?m)^bind = \".*\"$"), "bind = \"$bind\"")
196-
.replace(Regex("(?m)^anyhub = \".*\"$"), "anyhub = \"$hub\""))
199+
.replace(Regex("(?m)^bind = \".*\"$"), "bind = \"$bind\""))
197200
dir.resolve("forwarding.secret").writeText(secret)
198201
dir.resolve("cookie.secret").writeText(cookieSecret)
199202
}
200203
}
201204

202205
tasks.register<JavaExec>("runProxy") {
203206
group = "proxy"
204-
description = "Runs the staged proxy locally, in front of -PproxyHub (127.0.0.1:25565)."
207+
description = "Runs the staged proxy locally, in front of whatever hub the ipc session service names."
205208
dependsOn(prepareProxyRun)
206209
workingDir = proxyRunDir.get().asFile
207210
classpath = files(proxyRunDir.map { it.file("velocity.jar") })

‎bin/proxy-plugin/proxy/velocity.toml‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -71,17 +71,12 @@ bytes-per-second = -1
7171
# Increased to allow bigger Axiom pastes.
7272
decompressed-bytes-per-second = 67108864
7373

74+
# Backends are never configured here: ProxyPlugin connects every player to a server the session
75+
# service hands it, hub included.
7476
[servers]
75-
# Configure your servers here. Each key represents the server's name, and the value
76-
# represents the IP address of the server to connect to.
77-
anyhub = "mapmaker-hub.mapmaker.svc.cluster.local:25565"
78-
79-
# In what order we should try servers when a player logs in or is kicked from a server.
80-
try = ["anyhub"]
77+
try = []
8178

8279
[forced-hosts]
83-
# Configure your forced hosts here.
84-
"hollowcube.dev" = ["anyhub"]
8580

8681
[advanced]
8782
# How large a Minecraft packet has to be before we compress it. Setting this to zero will

0 commit comments

Comments
 (0)