diff --git a/SYSTEM_PARAMS.md b/SYSTEM_PARAMS.md new file mode 100644 index 0000000000000000000000000000000000000000..4e86d34aba241a6560c94a57c19db50be172e187 --- /dev/null +++ b/SYSTEM_PARAMS.md @@ -0,0 +1,18 @@ +## System parameters +System parameters refer to the parameters obtained directly through System.getproperty() in the code + +Param|Examples|Description of Associated Value +---|:---|:--- +os.name|'Mac OS X'|Operating system name +user.name|'hdfs'|User's account name +user.home|'/Users/hdfs'|User's home directory +user.dir|'/Users/hdfs/gitee_space/deepexi-daas-dolphinscheduler'|User's current working directory +file.separator|'/'|File separator ("/" on UNIX) +line.separator|'\n'|Line separator ("\n" on UNIX) +warmupIterations|10|JMH param. Number of warmup iterations +measureIterations|10|JMH param. Number of iterations +forkCount|2|JMH param. Number of forks +perfReportDir|-|JMH param. JMH report directory +jdk.lang.Process.allowAmbiguousCommands|true|allow ambiguous commands flag +netty.epoll.enable|true|Netty enable Linux epoll model flag +dolphin.scheduler.network.interface.preferred| |Preferred network interface for dolphin scheduler \ No newline at end of file diff --git a/SYSTEM_PARAMS_zh_CN.md b/SYSTEM_PARAMS_zh_CN.md new file mode 100644 index 0000000000000000000000000000000000000000..25f83d558f806b76a36d4f73faa8dd17de18e24e --- /dev/null +++ b/SYSTEM_PARAMS_zh_CN.md @@ -0,0 +1,18 @@ +## 系统参数 +系统参数，指的是代码中直接通过System.getProperty()获取的参数 + +参数|示例|描述 +---|:---:|:---:|:--- +os.name|'Mac OS X'|操作系统名称 +user.name|'squid'|操作系统用户名 +user.home|'/Users/squid'|操纵系统家目录 +user.dir|'/Users/squid/gitee_space/deepexi-daas-dolphinscheduler'|用户当前工作目录 +file.separator|'/'、 '\'|文件分割符，在UNIX系统中为'/' +line.separator|'\n'|换行符，在UNIX系统中为'\n' +warmupIterations|10|JMH预热次数 +measureIterations|10|JMH正式计量测试次数 +forkCount|2|JMH单次基准测试fork次数 +perfReportDir|-|JMH测试报告目录 +jdk.lang.Process.allowAmbiguousCommands|true|是否允许执行本地命令 +netty.epoll.enable|true|netty是否启用Linux Epoll模型 +dolphin.scheduler.network.interface.preferred||dolphin.scheduler首选网络接口 \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java index db46c5bdfb52e6d186f4400444897c15a74e36dc..2f94200cab78a19d34c5a14b837d4bf2f50dc6ef 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.LoggerService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.remote.utils.Host; @@ -100,7 +101,7 @@ public class LoggerServiceImpl implements LoggerService { String head = String.format(LOG_HEAD_FORMAT, taskInstance.getLogPath(), host, - Constants.SYSTEM_LINE_SEPARATOR); + OSUtils.getLineSeparator()); log.append(head); } @@ -128,7 +129,7 @@ public class LoggerServiceImpl implements LoggerService { byte[] head = String.format(LOG_HEAD_FORMAT, taskInstance.getLogPath(), host, - Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8); + OSUtils.getLineSeparator()).getBytes(StandardCharsets.UTF_8); return ArrayUtils.addAll(head, logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath())); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java index 1b05eb0e99188e31fa4d1038e051d50763421e14..ad40ecb3326a1124441ffed61c4007ae82676ed2 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java @@ -31,6 +31,8 @@ import java.io.*; import static org.junit.Assert.*; +import org.apache.dolphinscheduler.common.utils.OSUtils; + public class FileUtilsTest { private static final Logger logger = LoggerFactory.getLogger(FileUtilsTest.class); @@ -63,7 +65,7 @@ public class FileUtilsTest { public void testCopyFile() throws IOException { //Define dest file path - String destFilename = rootPath + System.getProperty("file.separator") + "data.txt"; + String destFilename = rootPath + OSUtils.getFileSeparator() + "data.txt"; logger.info("destFilename: "+destFilename); //Define InputStream for MultipartFile @@ -87,7 +89,7 @@ public class FileUtilsTest { public void testFile2Resource() throws IOException { //Define dest file path - String destFilename = rootPath + System.getProperty("file.separator") + "data.txt"; + String destFilename = rootPath + OSUtils.getFileSeparator() + "data.txt"; logger.info("destFilename: "+destFilename); //Define test resource diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 4320bd9710a9d59dd1055d1a5c3a62f60081f47b..a9d1c532fb6e4dbc22f6cffb0c83f786c1dfaa89 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -1024,11 +1024,6 @@ public final class Constants { public static final String END_TIME = "end time"; public static final String START_END_DATE = "startDate,endDate"; - /** - * system line separator - */ - public static final String SYSTEM_LINE_SEPARATOR = System.getProperty("line.separator"); - /** * net system properties */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/DolphinPluginManagerConfig.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/DolphinPluginManagerConfig.java index 7979868ef48f501eaabfd66d53dd75d202ff49ba..68e3da4f7b0ce59dfd99f65a96b27e7420885f06 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/DolphinPluginManagerConfig.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/DolphinPluginManagerConfig.java @@ -20,6 +20,8 @@ package org.apache.dolphinscheduler.common.plugin; import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import org.apache.dolphinscheduler.common.utils.OSUtils; + import java.io.File; import java.util.List; @@ -46,7 +48,7 @@ public class DolphinPluginManagerConfig { /** * Development, When AlertServer is running on IDE, AlertPluginLoad can load Alert Plugin from local Repository. */ - private String mavenLocalRepository = System.getProperty("user.home") + "/.m2/repository"; + private String mavenLocalRepository = OSUtils.getUserHome() + "/.m2/repository"; private List mavenRemoteRepository = ImmutableList.of("http://repo1.maven.org/maven2/"); File getInstalledPluginsDir() { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java index 9a24fdb407930f043be31212470b24a599be51c6..ba01deb15fb46598df40354d3a3dc3147dfa213b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.common.shell; +import org.apache.dolphinscheduler.common.utils.OSUtils; + import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -169,7 +171,7 @@ public abstract class AbstractShell { String line = errReader.readLine(); while((line != null) && !isInterrupted()) { errMsg.append(line); - errMsg.append(System.getProperty("line.separator")); + errMsg.append(OSUtils.getLineSeparator()); line = errReader.readLine(); } } catch(IOException ioe) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java index 4f0953f82445e2a442b414faea9de771d29aaa94..c2777203bf518a3a480e953c28a57f3682e7de13 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java @@ -260,7 +260,7 @@ public class NetUtils { } private static boolean isSpecifyNetworkInterface(NetworkInterface networkInterface) { - String preferredNetworkInterface = System.getProperty(DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE); + String preferredNetworkInterface = PropertyUtils.getString(DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE); return Objects.equals(networkInterface.getDisplayName(), preferredNetworkInterface); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index 3851f77e94717237789e85d4a9244f8135191bbf..c96ef9e25e64d5789e90a39840215cef3ed5279d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -381,7 +381,7 @@ public class OSUtils { */ public static String getGroup() throws IOException { if (isWindows()) { - String currentProcUserName = System.getProperty("user.name"); + String currentProcUserName = getUserName(); String result = exeCmd(String.format("net user \"%s\"", currentProcUserName)); String line = result.split("\n")[22]; String group = PATTERN.split(line)[1]; @@ -479,6 +479,51 @@ public class OSUtils { return System.getProperty("os.name"); } + /** + * get current OS user name + * + * @return current OS user name + */ + public static String getUserName() { + return System.getProperty("user.name"); + } + + /** + * get current OS user home + * + * @return current OS user home + */ + public static String getUserHome() { + return System.getProperty("user.home"); + } + + /** + * get current User's working directory + * + * @return current User's working directory + */ + public static String getUserDir() { + return System.getProperty("user.dir"); + } + + /** + * get file separator + * + * @return file separator + */ + public static String getFileSeparator() { + return System.getProperty("file.separator"); + } + + /** + * get line separator + * + * @return line separator + */ + public static String getLineSeparator() { + return System.getProperty("line.separator"); + } + /** * check memory and cpu usage * diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 726c799c72c375351b4783362a366a88ac8ad7fa..ab4e84d4378fb99f600e9cc190fb2e61ee676aba 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -86,3 +86,9 @@ sudo.enable=true # development state development.state=false + +# allow ambiguous commands flag +jdk.lang.Process.allowAmbiguousCommands = true + +# preferred network interface for dolphin scheduler, if not, keep this value empty +dolphin.scheduler.network.interface.preferred = diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index f0ffc52f68f23c87b617e7917716690301743999..5633ea4e2101531c78c215a2a541f98c8926086b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -43,7 +43,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); private static final String T_VERSION_NAME = "t_escheduler_version"; private static final String T_NEW_VERSION_NAME = "t_ds_version"; - private static final String rootDir = System.getProperty("user.dir"); + private static final String rootDir = OSUtils.getUserDir(); protected static final DataSource dataSource = getDataSource(); private static final DbType dbType = getCurrentDbType(); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java index 9ac1a9a72f6cc4b15cef4be0c407da0564915999..c51d95e63182179c6b46721d766d9f145d6db0fd 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java @@ -57,11 +57,6 @@ public class Constants { */ public static final String NETTY_EPOLL_ENABLE = System.getProperty("netty.epoll.enable", "true"); - /** - * OS Name - */ - public static final String OS_NAME = System.getProperty("os.name"); - /** * warm up time */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java index 89eb1f96079b0cb3226a614696c507f67e8fc423..06420eee109d3e7c5685e2aaa5d3566c0fcff63a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.remote.utils; +import org.apache.dolphinscheduler.common.utils.OSUtils; + import io.netty.channel.epoll.Epoll; import io.netty.channel.epoll.EpollServerSocketChannel; import io.netty.channel.epoll.EpollSocketChannel; @@ -34,7 +36,7 @@ public class NettyUtils { } public static boolean useEpoll() { - String osName = Constants.OS_NAME; + String osName = OSUtils.getOSName(); if (!osName.toLowerCase().contains("linux")) { return false; } diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/NettyUtilTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/NettyUtilTest.java index a3e13850ac636d517f39326dc650d8e0c2bd047d..879449eb8638e5b57d43b26751014f5e66563188 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/NettyUtilTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/NettyUtilTest.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.remote.utils; -import static org.apache.dolphinscheduler.remote.utils.Constants.OS_NAME; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.junit.Assert; import org.junit.Test; @@ -31,7 +31,7 @@ public class NettyUtilTest { @Test public void testUserEpoll() { - if (OS_NAME.toLowerCase().contains("linux") && Epoll.isAvailable()) { + if (OSUtils.getOSName().toLowerCase().contains("linux") && Epoll.isAvailable()) { Assert.assertTrue(NettyUtils.useEpoll()); } else { Assert.assertFalse(NettyUtils.useEpoll()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index 4d2ace663e9b6903857b4b2be518e9841b21f7bb..42ab37e7c4731cc75bd4453dd77ad4d4d19c0980 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -123,7 +123,7 @@ public class ProcessUtils { boolean allowAmbiguousCommands = false; if (security == null) { allowAmbiguousCommands = true; - String value = System.getProperty(LOCAL_PROCESS_EXEC); + String value = PropertyUtils.getString(LOCAL_PROCESS_EXEC); if (value != null) { allowAmbiguousCommands = !Constants.STRING_FALSE.equalsIgnoreCase(value); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java index f4e9080a689f9df61f6db4c9db9ffc935dc19c99..5d95860e271c5daa5264937887b8dd03f94a3b88 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.server.worker.task; +import org.apache.dolphinscheduler.common.utils.OSUtils; + import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -28,7 +30,7 @@ public class PythonCommandExecutorTest { @Test public void testGetPythonHome() { - String path = System.getProperty("user.dir") + "/script/env/dolphinscheduler_env.sh"; + String path = OSUtils.getUserDir() + "/script/env/dolphinscheduler_env.sh"; if (path.contains("dolphinscheduler-server/")) { path = path.replace("dolphinscheduler-server/", ""); } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java index 7ac23a3c4df1cc40802f3c61a0a8a3cf82df1c59..b73e774be3fae448c9203858a273390e14fe9f03 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.service.zk; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.service.exceptions.ServiceException; @@ -117,7 +118,7 @@ public class ZKServer { * @param port The port to listen on */ public void startLocalZkServer(final int port) { - String zkDataDir = System.getProperty("user.dir") + (StringUtils.isEmpty(prefix) ? StringUtils.EMPTY : ("/" + prefix)) + "/zookeeper_data"; + String zkDataDir = OSUtils.getUserDir() + (StringUtils.isEmpty(prefix) ? StringUtils.EMPTY : ("/" + prefix)) + "/zookeeper_data"; File file = new File(zkDataDir); if (file.exists()) { logger.warn("The path of zk server exists");