Skip to content

Commit 2d63804

Browse files
Tsukilcbitflicker64imbajin
authored
refactor(server): disable legacy master-worker scheduler logic (#3082)
This PR soft-disables the pre-PD master-worker task scheduling path while keeping old configs and task data upgrade-safe. Scheduler selection is now backend-driven: `hstore` uses `DistributedTaskScheduler`, and other backends use the local `StandardTaskScheduler`. --------- Co-authored-by: Himanshu Verma <himnshuverma10152006@gmail.com> Co-authored-by: imbajin <jin@apache.org>
1 parent 03e6b8e commit 2d63804

23 files changed

Lines changed: 1107 additions & 852 deletions

File tree

‎hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/hugegraph.properties.template‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@ store=hugegraph
4545
pd.peers=$PD_PEERS_LIST$
4646

4747
# task config
48-
task.scheduler_type=local
4948
task.schedule_period=10
5049
task.retry=0
5150
task.wait_timeout=10

‎hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/config/ServerOptions.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -556,9 +556,9 @@ public class ServerOptions extends OptionHolder {
556556
public static final ConfigOption<String> SERVER_ID =
557557
new ConfigOption<>(
558558
"server.id",
559-
"The id of hugegraph-server.",
560-
disallowEmpty(),
561-
"server-1"
559+
"The optional legacy id of hugegraph-server.",
560+
null,
561+
""
562562
);
563563
public static final ConfigOption<String> SERVER_ROLE =
564564
new ConfigOption<>(

‎hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java‎

Lines changed: 50 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@
5858
import org.apache.hugegraph.backend.BackendException;
5959
import org.apache.hugegraph.backend.cache.Cache;
6060
import org.apache.hugegraph.backend.cache.CacheManager;
61-
import org.apache.hugegraph.backend.id.IdGenerator;
6261
import org.apache.hugegraph.backend.store.AbstractBackendStoreProvider;
6362
import org.apache.hugegraph.backend.store.BackendStoreInfo;
6463
import org.apache.hugegraph.config.ConfigOption;
@@ -68,6 +67,7 @@
6867
import org.apache.hugegraph.config.TypedOption;
6968
import org.apache.hugegraph.event.EventHub;
7069
import org.apache.hugegraph.exception.ExistedException;
70+
import org.apache.hugegraph.exception.NotFoundException;
7171
import org.apache.hugegraph.exception.NotSupportException;
7272
import org.apache.hugegraph.io.HugeGraphSONModule;
7373
import org.apache.hugegraph.k8s.K8sDriver;
@@ -195,8 +195,6 @@ public final class GraphManager {
195195
public GraphManager(HugeConfig conf, EventHub hub) {
196196
LOG.info("Init graph manager");
197197
E.checkArgumentNotNull(conf, "The config can't be null");
198-
String server = conf.get(ServerOptions.SERVER_ID);
199-
String role = conf.get(ServerOptions.SERVER_ROLE);
200198

201199
this.config = conf;
202200
this.url = conf.get(ServerOptions.REST_SERVER_URL);
@@ -206,10 +204,6 @@ public GraphManager(HugeConfig conf, EventHub hub) {
206204
conf.get(ServerOptions.SERVER_DEPLOY_IN_K8S);
207205
this.startIgnoreSingleGraphError = conf.get(
208206
ServerOptions.SERVER_START_IGNORE_SINGLE_GRAPH_ERROR);
209-
E.checkArgument(server != null && !server.isEmpty(),
210-
"The server name can't be null or empty");
211-
E.checkArgument(role != null && !role.isEmpty(),
212-
"The server role can't be null or empty");
213207
this.graphsDir = conf.get(ServerOptions.GRAPHS);
214208
this.cluster = conf.get(ServerOptions.CLUSTER);
215209
this.graphSpaces = new ConcurrentHashMap<>();
@@ -1557,6 +1551,9 @@ private void loadGraph(String name, String graphConfPath) {
15571551
String raftGroupPeers = this.conf.get(ServerOptions.RAFT_GROUP_PEERS);
15581552
config.addProperty(ServerOptions.RAFT_GROUP_PEERS.name(),
15591553
raftGroupPeers);
1554+
1555+
this.transferPdPeersConfig(config);
1556+
15601557
this.transferRoleWorkerConfig(config);
15611558

15621559
Graph graph = GraphFactory.open(config);
@@ -1575,6 +1572,19 @@ private void loadGraph(String name, String graphConfPath) {
15751572
}
15761573
}
15771574

1575+
private void transferPdPeersConfig(HugeConfig config) {
1576+
if (config.containsKey(CoreOptions.PD_PEERS.name())) {
1577+
return;
1578+
}
1579+
1580+
String backend = config.get(CoreOptions.BACKEND);
1581+
boolean needPdPeers = this.conf.get(ServerOptions.USE_PD) ||
1582+
StringUtils.equalsIgnoreCase("hstore", backend);
1583+
if (needPdPeers) {
1584+
config.addProperty(CoreOptions.PD_PEERS.name(), this.pdPeers);
1585+
}
1586+
}
1587+
15781588
private void transferRoleWorkerConfig(HugeConfig config) {
15791589
config.setProperty(RoleElectionOptions.NODE_EXTERNAL_URL.name(),
15801590
this.conf.get(ServerOptions.REST_SERVER_URL));
@@ -1635,23 +1645,21 @@ private void checkBackendVersionOrExit(HugeConfig config) {
16351645
}
16361646

16371647
private void initNodeRole() {
1638-
String id = config.get(ServerOptions.SERVER_ID);
1648+
boolean enableRoleElection = config.get(
1649+
ServerOptions.ENABLE_SERVER_ROLE_ELECTION);
1650+
if (enableRoleElection) {
1651+
LOG.warn("The server.role_election option is deprecated and no " +
1652+
"longer supported (removed with server_info persistence). " +
1653+
"The configured server.role is still used for local node " +
1654+
"role initialization. Set server.role_election=false to " +
1655+
"suppress this warning.");
1656+
}
1657+
16391658
String role = config.get(ServerOptions.SERVER_ROLE);
1640-
E.checkArgument(StringUtils.isNotEmpty(id),
1641-
"The server name can't be null or empty");
16421659
E.checkArgument(StringUtils.isNotEmpty(role),
16431660
"The server role can't be null or empty");
16441661

16451662
NodeRole nodeRole = NodeRole.valueOf(role.toUpperCase());
1646-
boolean supportRoleElection = !nodeRole.computer() &&
1647-
this.supportRoleElection() &&
1648-
config.get(ServerOptions.ENABLE_SERVER_ROLE_ELECTION);
1649-
if (supportRoleElection) {
1650-
// Init any server as Worker role, then do role election
1651-
nodeRole = NodeRole.WORKER;
1652-
}
1653-
1654-
this.globalNodeRoleInfo.initNodeId(IdGenerator.of(id));
16551663
this.globalNodeRoleInfo.initNodeRole(nodeRole);
16561664
}
16571665

@@ -1663,7 +1671,7 @@ private void serverStarted(HugeConfig conf) {
16631671
}
16641672
if (!this.globalNodeRoleInfo.nodeRole().computer() && this.supportRoleElection() &&
16651673
config.get(ServerOptions.ENABLE_SERVER_ROLE_ELECTION)) {
1666-
this.initRoleStateMachine();
1674+
LOG.info("Skip role state machine init (deprecated with server_info)");
16671675
}
16681676
}
16691677

@@ -1937,26 +1945,29 @@ public Set<String> getServiceUrls(String graphSpace, String service,
19371945
public HugeGraph graph(String graphSpace, String name) {
19381946
String key = String.join(DELIMITER, graphSpace, name);
19391947
Graph graph = this.graphs.get(key);
1940-
if (graph == null && isPDEnabled()) {
1941-
Map<String, Map<String, Object>> configs =
1942-
this.metaManager.graphConfigs(graphSpace);
1943-
// If current server registered graph space is not DEFAULT, only load graph creation
1944-
// under registered graph space
1945-
if (!configs.containsKey(key) ||
1946-
(!"DEFAULT".equals(this.serviceGraphSpace) &&
1947-
!graphSpace.equals(this.serviceGraphSpace))) {
1948-
return null;
1948+
if (graph == null) {
1949+
if (isPDEnabled()) {
1950+
Map<String, Map<String, Object>> configs =
1951+
this.metaManager.graphConfigs(graphSpace);
1952+
// If current server registered graph space is not DEFAULT, only load graph creation
1953+
// under registered graph space
1954+
if (!configs.containsKey(key) ||
1955+
(!"DEFAULT".equals(this.serviceGraphSpace) &&
1956+
!graphSpace.equals(this.serviceGraphSpace))) {
1957+
return null;
1958+
}
1959+
Map<String, Object> config = configs.get(key);
1960+
String creator = String.valueOf(config.get("creator"));
1961+
Date createTime = parseDate(config.get("create_time"));
1962+
Date updateTime = parseDate(config.get("update_time"));
1963+
HugeGraph graph1 = this.createGraph(graphSpace, name,
1964+
creator, config, false);
1965+
graph1.createTime(createTime);
1966+
graph1.updateTime(updateTime);
1967+
this.graphs.put(key, graph1);
1968+
return graph1;
19491969
}
1950-
Map<String, Object> config = configs.get(key);
1951-
String creator = String.valueOf(config.get("creator"));
1952-
Date createTime = parseDate(config.get("create_time"));
1953-
Date updateTime = parseDate(config.get("update_time"));
1954-
HugeGraph graph1 = this.createGraph(graphSpace, name,
1955-
creator, config, false);
1956-
graph1.createTime(createTime);
1957-
graph1.updateTime(updateTime);
1958-
this.graphs.put(key, graph1);
1959-
return graph1;
1970+
throw new NotFoundException(String.format("Graph '%s' does not exist", name));
19601971
} else if (graph instanceof HugeGraph) {
19611972
return (HugeGraph) graph;
19621973
}

‎hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,6 @@ public class StandardHugeGraph implements HugeGraph {
177177
private final BackendStoreProvider storeProvider;
178178
private final TinkerPopTransaction tx;
179179
private final RamTable ramtable;
180-
private final String schedulerType;
181180
private volatile boolean started;
182181
private volatile boolean closed;
183182
private volatile GraphMode mode;
@@ -226,11 +225,18 @@ public StandardHugeGraph(HugeConfig config) {
226225

227226
this.taskManager = TaskManager.instance();
228227
this.name = config.get(CoreOptions.STORE);
228+
229+
// Keep old config files upgrade-safe while ignoring the legacy scheduler.
230+
if (config.containsKey("task.scheduler_type")) {
231+
LOG.warn("Config key 'task.scheduler_type' is deprecated and " +
232+
"ignored. The scheduler is auto-selected by backend " +
233+
"type (hstore -> distributed, others -> local).");
234+
}
235+
229236
this.started = false;
230237
this.closed = false;
231238
this.mode = GraphMode.NONE;
232239
this.readMode = GraphReadMode.OLTP_ONLY;
233-
this.schedulerType = config.get(CoreOptions.SCHEDULER_TYPE);
234240

235241
// Init process-wide static configs before lock, so that validation
236242
// failures won't leave stale lock groups in LockManager.
@@ -323,6 +329,7 @@ public String backend() {
323329
return this.storeProvider.type();
324330
}
325331

332+
@Override
326333
public BackendStoreInfo backendStoreInfo() {
327334
// Just for trigger Tx.getOrNewTransaction, then load 3 stores
328335
// TODO: pass storeProvider.metaStore()
@@ -340,11 +347,10 @@ public void serverStarted(GlobalMasterInfo nodeInfo) {
340347
LOG.info("Init system info for graph '{}'", this.spaceGraphName());
341348
this.initSystemInfo();
342349

343-
LOG.info("Init server info [{}-{}] for graph '{}'...",
344-
nodeInfo.nodeId(), nodeInfo.nodeRole(), this.spaceGraphName());
345-
this.serverInfoManager().initServerInfo(nodeInfo);
346-
347-
this.initRoleStateMachine(nodeInfo.nodeId());
350+
if (nodeInfo != null && nodeInfo.nodeId() != null) {
351+
this.serverInfoManager().initServerInfo(nodeInfo);
352+
this.initRoleStateMachine(nodeInfo.nodeId());
353+
}
348354

349355
// TODO: check necessary?
350356
LOG.info("Check olap property-key tables for graph '{}'", this.spaceGraphName());
@@ -473,6 +479,7 @@ public void updateTime(Date updateTime) {
473479
this.updateTime = updateTime;
474480
}
475481

482+
@Override
476483
public void waitStarted() {
477484
// Just for trigger Tx.getOrNewTransaction, then load 3 stores
478485
this.schemaTransaction();
@@ -489,9 +496,7 @@ public void initBackend() {
489496
try {
490497
this.storeProvider.init();
491498
/*
492-
* NOTE: The main goal is to write the serverInfo to the central
493-
* node, such as etcd, and also create the system schema in memory,
494-
* which has no side effects
499+
* NOTE: Create system schema in memory, which has no side effects.
495500
*/
496501
this.initSystemInfo();
497502
} finally {
@@ -532,8 +537,7 @@ public void truncateBackend() {
532537
LockUtil.lock(this.spaceGraphName(), LockUtil.GRAPH_LOCK);
533538
try {
534539
this.storeProvider.truncate();
535-
// TODO: remove this after serverinfo saved in etcd
536-
this.serverStarted(this.serverInfoManager().globalNodeRoleInfo());
540+
this.serverStarted(null);
537541
} finally {
538542
LockUtil.unlock(this.spaceGraphName(), LockUtil.GRAPH_LOCK);
539543
}
@@ -555,7 +559,6 @@ public KvStore kvStore() {
555559
public void initSystemInfo() {
556560
try {
557561
this.taskScheduler().init();
558-
this.serverInfoManager().init();
559562
this.authManager().init();
560563
} finally {
561564
this.closeTx();
@@ -1640,7 +1643,9 @@ public <T> void submitEphemeralJob(EphemeralJob<T> job) {
16401643

16411644
@Override
16421645
public String schedulerType() {
1643-
return StandardHugeGraph.this.schedulerType;
1646+
// Use distributed scheduler for hstore backend, otherwise use local
1647+
// After the merger of rocksdb and hstore, consider whether to change this logic
1648+
return StandardHugeGraph.this.isHstore() ? "distributed" : "local";
16441649
}
16451650
}
16461651

‎hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/config/CoreOptions.java‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -313,13 +313,7 @@ public class CoreOptions extends OptionHolder {
313313
rangeInt(1, 500),
314314
1
315315
);
316-
public static final ConfigOption<String> SCHEDULER_TYPE =
317-
new ConfigOption<>(
318-
"task.scheduler_type",
319-
"The type of scheduler used in distribution system.",
320-
allowValues("local", "distributed"),
321-
"local"
322-
);
316+
323317
public static final ConfigOption<Boolean> TASK_SYNC_DELETION =
324318
new ConfigOption<>(
325319
"task.sync_deletion",

‎hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/masterelection/GlobalMasterInfo.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import org.apache.hugegraph.type.define.NodeRole;
2323
import org.apache.hugegraph.util.E;
2424

25-
// TODO: rename to GlobalNodeRoleInfo
25+
// TODO: We need to completely delete the startup of master-worker
2626
public final class GlobalMasterInfo {
2727

2828
private static final NodeInfo NO_MASTER = new NodeInfo(false, "");

‎hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/masterelection/StandardRoleListener.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ public class StandardRoleListener implements RoleListener {
3636
public StandardRoleListener(TaskManager taskManager,
3737
GlobalMasterInfo roleInfo) {
3838
this.taskManager = taskManager;
39-
this.taskManager.enableRoleElection();
4039
this.roleInfo = roleInfo;
4140
this.selfIsMaster = false;
4241
}

0 commit comments

Comments
 (0)