Skip to content

Commit 4bec792

Browse files
brave-leebrave-lee
authored
[Fix][Common] rewrite code generate,fix bit shift (apache#6914)
* rewrite code generate,fix bit shift * fix ut * add algorithm from licenses file * add algorithm from licenses file * add algorithm from licenses file * add algorithm from licenses file * add algorithm from licenses file * fix ut
1 parent 595e484 commit 4bec792

File tree

17 files changed

+142
-139
lines changed

17 files changed

+142
-139
lines changed

Diff for: ‎.licenserc.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ header:
2626
- LICENSE
2727
- DISCLAIMER
2828
- dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java
29+
- dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java
2930
- mvnw.cmd
3031
- dolphinscheduler-dao/src/main/resources/sql/soft_version
3132
- .mvn

Diff for: ‎LICENSE

+1-1
Original file line numberDiff line numberDiff line change
@@ -219,4 +219,4 @@ The text of each license is the standard Apache 2.0 license.
219219
DolphinPluginClassLoader from https://github.com/prestosql/presto Apache 2.0
220220
DolphinPluginDiscovery from https://github.com/prestosql/presto Apache 2.0
221221
DolphinPluginLoader from https://github.com/prestosql/presto Apache 2.0
222-
222+
CodeGenerateUtils from https://github.com/twitter-archive/snowflake/tree/snowflake-2010 Apache 2.0

Diff for: ‎dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@
2323
import org.apache.dolphinscheduler.api.utils.PageInfo;
2424
import org.apache.dolphinscheduler.api.utils.Result;
2525
import org.apache.dolphinscheduler.common.Constants;
26+
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
27+
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
2628
import org.apache.dolphinscheduler.common.utils.JSONUtils;
27-
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
28-
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
2929
import org.apache.dolphinscheduler.dao.entity.Environment;
3030
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
3131
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
@@ -115,9 +115,9 @@ public Map<String, Object> createEnvironment(User loginUser, String name, String
115115
env.setUpdateTime(new Date());
116116
long code = 0L;
117117
try {
118-
code = SnowFlakeUtils.getInstance().nextId();
118+
code = CodeGenerateUtils.getInstance().genCode();
119119
env.setCode(code);
120-
} catch (SnowFlakeException e) {
120+
} catch (CodeGenerateException e) {
121121
logger.error("Environment code get error, ", e);
122122
}
123123
if (code == 0L) {

Diff for: ‎dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

+10-10
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,10 @@
4040
import org.apache.dolphinscheduler.common.model.TaskNode;
4141
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
4242
import org.apache.dolphinscheduler.common.thread.Stopper;
43+
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
44+
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
4345
import org.apache.dolphinscheduler.common.utils.DateUtils;
4446
import org.apache.dolphinscheduler.common.utils.JSONUtils;
45-
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
46-
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
4747
import org.apache.dolphinscheduler.dao.entity.DagData;
4848
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
4949
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
@@ -222,8 +222,8 @@ public Map<String, Object> createProcessDefinition(User loginUser,
222222
}
223223
long processDefinitionCode;
224224
try {
225-
processDefinitionCode = SnowFlakeUtils.getInstance().nextId();
226-
} catch (SnowFlakeException e) {
225+
processDefinitionCode = CodeGenerateUtils.getInstance().genCode();
226+
} catch (CodeGenerateException e) {
227227
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS);
228228
return result;
229229
}
@@ -874,8 +874,8 @@ private boolean checkAndImport(User loginUser, long projectCode, Map<String, Obj
874874
processDefinition.setProjectCode(projectCode);
875875
processDefinition.setUserId(loginUser.getId());
876876
try {
877-
processDefinition.setCode(SnowFlakeUtils.getInstance().nextId());
878-
} catch (SnowFlakeException e) {
877+
processDefinition.setCode(CodeGenerateUtils.getInstance().genCode());
878+
} catch (CodeGenerateException e) {
879879
putMsg(result, Status.CREATE_PROCESS_DEFINITION_ERROR);
880880
return false;
881881
}
@@ -894,10 +894,10 @@ private boolean checkAndImport(User loginUser, long projectCode, Map<String, Obj
894894
taskDefinitionLog.setOperator(loginUser.getId());
895895
taskDefinitionLog.setOperateTime(now);
896896
try {
897-
long code = SnowFlakeUtils.getInstance().nextId();
897+
long code = CodeGenerateUtils.getInstance().genCode();
898898
taskCodeMap.put(taskDefinitionLog.getCode(), code);
899899
taskDefinitionLog.setCode(code);
900-
} catch (SnowFlakeException e) {
900+
} catch (CodeGenerateException e) {
901901
logger.error("Task code get error, ", e);
902902
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code");
903903
return false;
@@ -1363,8 +1363,8 @@ private void doBatchOperateProcessDefinition(User loginUser,
13631363
processDefinition.setProjectCode(targetProjectCode);
13641364
if (isCopy) {
13651365
try {
1366-
processDefinition.setCode(SnowFlakeUtils.getInstance().nextId());
1367-
} catch (SnowFlakeException e) {
1366+
processDefinition.setCode(CodeGenerateUtils.getInstance().genCode());
1367+
} catch (CodeGenerateException e) {
13681368
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS);
13691369
throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS);
13701370
}

Diff for: ‎dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525
import org.apache.dolphinscheduler.api.utils.Result;
2626
import org.apache.dolphinscheduler.common.Constants;
2727
import org.apache.dolphinscheduler.common.enums.UserType;
28-
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
29-
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
28+
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
29+
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
3030
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
3131
import org.apache.dolphinscheduler.dao.entity.Project;
3232
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
@@ -97,14 +97,14 @@ public Map<String, Object> createProject(User loginUser, String name, String des
9797
project = Project
9898
.newBuilder()
9999
.name(name)
100-
.code(SnowFlakeUtils.getInstance().nextId())
100+
.code(CodeGenerateUtils.getInstance().genCode())
101101
.description(desc)
102102
.userId(loginUser.getId())
103103
.userName(loginUser.getUserName())
104104
.createTime(now)
105105
.updateTime(now)
106106
.build();
107-
} catch (SnowFlakeException e) {
107+
} catch (CodeGenerateException e) {
108108
putMsg(result, Status.CREATE_PROJECT_ERROR);
109109
return result;
110110
}

Diff for: ‎dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@
2727
import org.apache.dolphinscheduler.common.Constants;
2828
import org.apache.dolphinscheduler.common.enums.Flag;
2929
import org.apache.dolphinscheduler.common.enums.ReleaseState;
30+
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
31+
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
3032
import org.apache.dolphinscheduler.common.utils.JSONUtils;
31-
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
32-
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
3333
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
3434
import org.apache.dolphinscheduler.dao.entity.Project;
3535
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
@@ -431,9 +431,9 @@ public Map<String, Object> genTaskCodeList(Integer genNum) {
431431
List<Long> taskCodes = new ArrayList<>();
432432
try {
433433
for (int i = 0; i < genNum; i++) {
434-
taskCodes.add(SnowFlakeUtils.getInstance().nextId());
434+
taskCodes.add(CodeGenerateUtils.getInstance().genCode());
435435
}
436-
} catch (SnowFlakeException e) {
436+
} catch (CodeGenerateException e) {
437437
logger.error("Task code get error, ", e);
438438
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code");
439439
}

Diff for: ‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/** Copyright 2010-2012 Twitter, Inc.*/
2+
3+
package org.apache.dolphinscheduler.common.utils;
4+
5+
import java.net.InetAddress;
6+
import java.net.UnknownHostException;
7+
import java.util.Objects;
8+
9+
/**
10+
* Rewriting based on Twitter snowflake algorithm
11+
*/
12+
public class CodeGenerateUtils {
13+
// start timestamp
14+
private static final long START_TIMESTAMP = 1609430400000L; //2021-01-01 00:00:00
15+
// Each machine generates 32 in the same millisecond
16+
private static final long LOW_DIGIT_BIT = 5L;
17+
private static final long MIDDLE_BIT = 2L;
18+
private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT);
19+
// The displacement to the left
20+
private static final long MIDDLE_LEFT = LOW_DIGIT_BIT;
21+
private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MIDDLE_BIT;
22+
private final long machineHash;
23+
private long lowDigit = 0L;
24+
private long recordMillisecond = -1L;
25+
26+
private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis();
27+
private static final long SYSTEM_NANOTIME = System.nanoTime();
28+
29+
private CodeGenerateUtils() throws CodeGenerateException {
30+
try {
31+
this.machineHash = Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % (2 << (MIDDLE_BIT - 1));
32+
} catch (UnknownHostException e) {
33+
throw new CodeGenerateException(e.getMessage());
34+
}
35+
}
36+
37+
private static CodeGenerateUtils instance = null;
38+
39+
public static synchronized CodeGenerateUtils getInstance() throws CodeGenerateException {
40+
if (instance == null) {
41+
instance = new CodeGenerateUtils();
42+
}
43+
return instance;
44+
}
45+
46+
public synchronized long genCode() throws CodeGenerateException {
47+
long nowtMillisecond = systemMillisecond();
48+
if (nowtMillisecond < recordMillisecond) {
49+
throw new CodeGenerateException("New code exception because time is set back.");
50+
}
51+
if (nowtMillisecond == recordMillisecond) {
52+
lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT;
53+
if (lowDigit == 0L) {
54+
while (nowtMillisecond <= recordMillisecond) {
55+
nowtMillisecond = systemMillisecond();
56+
}
57+
}
58+
} else {
59+
lowDigit = 0L;
60+
}
61+
recordMillisecond = nowtMillisecond;
62+
return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << MIDDLE_LEFT | lowDigit;
63+
}
64+
65+
private long systemMillisecond() {
66+
return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000;
67+
}
68+
69+
public static class CodeGenerateException extends Exception {
70+
public CodeGenerateException(String message) {
71+
super(message);
72+
}
73+
}
74+
}

Diff for: ‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtils.java

-94
This file was deleted.

Diff for: ‎dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtilsTest.java renamed to ‎dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java

+6-6
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,14 @@
2222
import org.junit.Assert;
2323
import org.junit.Test;
2424

25-
public class SnowFlakeUtilsTest {
25+
public class CodeGenerateUtilsTest {
2626
@Test
27-
public void testNoGenerateDuplicateId() throws SnowFlakeUtils.SnowFlakeException {
28-
HashSet<Long> existsSnowFlakeId = new HashSet<>();
27+
public void testNoGenerateDuplicateCode() throws CodeGenerateUtils.CodeGenerateException {
28+
HashSet<Long> existsCode = new HashSet<>();
2929
for (int i = 0; i < 100; i++) {
30-
Long currentId = SnowFlakeUtils.getInstance().nextId();
31-
Assert.assertFalse(existsSnowFlakeId.contains(currentId));
32-
existsSnowFlakeId.add(currentId);
30+
Long currentCode = CodeGenerateUtils.getInstance().genCode();
31+
Assert.assertFalse(existsCode.contains(currentCode));
32+
existsCode.add(currentCode);
3333
}
3434
}
3535
}

Diff for: ‎dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@
2020
import org.apache.dolphinscheduler.common.Constants;
2121
import org.apache.dolphinscheduler.common.enums.Flag;
2222
import org.apache.dolphinscheduler.common.enums.ReleaseState;
23+
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
2324
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
24-
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
2525
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
2626

2727
import java.sql.Connection;
@@ -110,7 +110,7 @@ public List<ProcessDefinition> queryProcessDefinition(Connection conn) {
110110
processDefinition.setId(rs.getInt(1));
111111
long code = rs.getLong(2);
112112
if (code == 0L) {
113-
code = SnowFlakeUtils.getInstance().nextId();
113+
code = CodeGenerateUtils.getInstance().genCode();
114114
}
115115
processDefinition.setCode(code);
116116
processDefinition.setVersion(Constants.VERSION_FIRST);

Diff for: ‎dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProjectDao.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717

1818
package org.apache.dolphinscheduler.dao.upgrade;
1919

20+
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
2021
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
21-
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
2222

2323
import java.sql.Connection;
2424
import java.sql.PreparedStatement;
@@ -51,7 +51,7 @@ public Map<Integer, Long> queryAllProject(Connection conn) {
5151
Integer id = rs.getInt(1);
5252
long code = rs.getLong(2);
5353
if (code == 0L) {
54-
code = SnowFlakeUtils.getInstance().nextId();
54+
code = CodeGenerateUtils.getInstance().genCode();
5555
}
5656
projectMap.put(id, code);
5757
}

Diff for: ‎dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,11 @@
2525
import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
2626
import org.apache.dolphinscheduler.common.process.ResourceInfo;
2727
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
28+
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
2829
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
2930
import org.apache.dolphinscheduler.common.utils.JSONUtils;
3031
import org.apache.dolphinscheduler.common.utils.SchemaUtils;
3132
import org.apache.dolphinscheduler.common.utils.ScriptRunner;
32-
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
3333
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
3434
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
3535
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
@@ -673,7 +673,7 @@ private void splitProcessDefinitionJson(List<ProcessDefinition> processDefinitio
673673
String name = task.get("name").asText();
674674
taskDefinitionLog.setName(name);
675675
taskDefinitionLog.setWorkerGroup(task.get("workerGroup").asText());
676-
long taskCode = SnowFlakeUtils.getInstance().nextId();
676+
long taskCode = CodeGenerateUtils.getInstance().genCode();
677677
taskDefinitionLog.setCode(taskCode);
678678
taskDefinitionLog.setVersion(Constants.VERSION_FIRST);
679679
taskDefinitionLog.setProjectCode(processDefinition.getProjectCode());

0 commit comments

Comments
 (0)