From a50194c056a1a6f44a2d3cd1920d8fba28c751d5 Mon Sep 17 00:00:00 2001 From: Squid <2824638304@qq.com> Date: Fri, 4 Jun 2021 09:26:06 +0800 Subject: [PATCH 1/2] add: system params file --- SYSTEM_PARAMS.md | 18 ++++++++++++++++++ SYSTEM_PARAMS_zh_CN.md | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 SYSTEM_PARAMS.md create mode 100644 SYSTEM_PARAMS_zh_CN.md diff --git a/SYSTEM_PARAMS.md b/SYSTEM_PARAMS.md new file mode 100644 index 0000000000..f37a413d5d --- /dev/null +++ b/SYSTEM_PARAMS.md @@ -0,0 +1,18 @@ +## System parameters +System parameters refer to the parameters obtained directly through System.getproperty() in the code + +Param|Examples|Description of Associated Value +---|:---|:--- +os.name|'Mac OS X'|Operating system name +user.name|'hdfs'|User's account name +user.home|'/Users/hdfs'|User's home directory +user.dir|'/Users/hdfs/gitee_space/deepexi-daas-dolphinscheduler'|User's current working directory +file.separator|'/'|File separator ("/" on UNIX) +line.separator|'\n'|Line separator ("\n" on UNIX) +warmupIterations|10|JMH param. Number of warmup iterations +measureIterations|10|JMH param. Number of iterations +forkCount|2|JMH param. Number of forks +perfReportDir|-|JMH param. JMH report directory +jdk.lang.Process.allowAmbiguousCommands|true|allow ambiguous commands flag +netty.epoll.enable|true|Netty enable Linux epoll model flag +dolphin.scheduler.network.interface.preferred||Preferred network interface for dolphin scheduler \ No newline at end of file diff --git a/SYSTEM_PARAMS_zh_CN.md b/SYSTEM_PARAMS_zh_CN.md new file mode 100644 index 0000000000..25f83d558f --- /dev/null +++ b/SYSTEM_PARAMS_zh_CN.md @@ -0,0 +1,18 @@ +## 系统参数 +系统参数，指的是代码中直接通过System.getProperty()获取的参数 + +参数|示例|描述 +---|:---:|:---:|:--- +os.name|'Mac OS X'|操作系统名称 +user.name|'squid'|操作系统用户名 +user.home|'/Users/squid'|操纵系统家目录 +user.dir|'/Users/squid/gitee_space/deepexi-daas-dolphinscheduler'|用户当前工作目录 +file.separator|'/'、 '\'|文件分割符，在UNIX系统中为'/' +line.separator|'\n'|换行符，在UNIX系统中为'\n' +warmupIterations|10|JMH预热次数 +measureIterations|10|JMH正式计量测试次数 +forkCount|2|JMH单次基准测试fork次数 +perfReportDir|-|JMH测试报告目录 +jdk.lang.Process.allowAmbiguousCommands|true|是否允许执行本地命令 +netty.epoll.enable|true|netty是否启用Linux Epoll模型 +dolphin.scheduler.network.interface.preferred||dolphin.scheduler首选网络接口 \ No newline at end of file -- Gitee From d2c2e25f847628db8dca0304e9cd542fc48a0410 Mon Sep 17 00:00:00 2001 From: Squid <2824638304@qq.com> Date: Tue, 8 Jun 2021 21:42:53 +0800 Subject: [PATCH 2/2] Update:replace 'System.getProperty()' method --- SYSTEM_PARAMS.md | 2 +- .../api/service/impl/LoggerServiceImpl.java | 5 +- .../api/utils/FileUtilsTest.java | 6 ++- .../dolphinscheduler/common/Constants.java | 5 -- .../plugin/DolphinPluginManagerConfig.java | 4 +- .../common/shell/AbstractShell.java | 4 +- .../common/utils/NetUtils.java | 2 +- .../common/utils/OSUtils.java | 47 ++++++++++++++++++- .../src/main/resources/common.properties | 6 +++ .../dao/upgrade/UpgradeDao.java | 2 +- .../remote/utils/Constants.java | 5 -- .../remote/utils/NettyUtils.java | 4 +- .../remote/utils/NettyUtilTest.java | 4 +- .../server/utils/ProcessUtils.java | 2 +- .../task/PythonCommandExecutorTest.java | 4 +- .../dolphinscheduler/service/zk/ZKServer.java | 3 +- 16 files changed, 79 insertions(+), 26 deletions(-) diff --git a/SYSTEM_PARAMS.md b/SYSTEM_PARAMS.md index f37a413d5d..4e86d34aba 100644 --- a/SYSTEM_PARAMS.md +++ b/SYSTEM_PARAMS.md @@ -15,4 +15,4 @@ forkCount|2|JMH param. Number of forks perfReportDir|-|JMH param. JMH report directory jdk.lang.Process.allowAmbiguousCommands|true|allow ambiguous commands flag netty.epoll.enable|true|Netty enable Linux epoll model flag -dolphin.scheduler.network.interface.preferred||Preferred network interface for dolphin scheduler \ No newline at end of file +dolphin.scheduler.network.interface.preferred| |Preferred network interface for dolphin scheduler \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java index db46c5bdfb..2f94200cab 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.LoggerService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.remote.utils.Host; @@ -100,7 +101,7 @@ public class LoggerServiceImpl implements LoggerService { String head = String.format(LOG_HEAD_FORMAT, taskInstance.getLogPath(), host, - Constants.SYSTEM_LINE_SEPARATOR); + OSUtils.getLineSeparator()); log.append(head); } @@ -128,7 +129,7 @@ public class LoggerServiceImpl implements LoggerService { byte[] head = String.format(LOG_HEAD_FORMAT, taskInstance.getLogPath(), host, - Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8); + OSUtils.getLineSeparator()).getBytes(StandardCharsets.UTF_8); return ArrayUtils.addAll(head, logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath())); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java index 1b05eb0e99..ad40ecb332 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java @@ -31,6 +31,8 @@ import java.io.*; import static org.junit.Assert.*; +import org.apache.dolphinscheduler.common.utils.OSUtils; + public class FileUtilsTest { private static final Logger logger = LoggerFactory.getLogger(FileUtilsTest.class); @@ -63,7 +65,7 @@ public class FileUtilsTest { public void testCopyFile() throws IOException { //Define dest file path - String destFilename = rootPath + System.getProperty("file.separator") + "data.txt"; + String destFilename = rootPath + OSUtils.getFileSeparator() + "data.txt"; logger.info("destFilename: "+destFilename); //Define InputStream for MultipartFile @@ -87,7 +89,7 @@ public class FileUtilsTest { public void testFile2Resource() throws IOException { //Define dest file path - String destFilename = rootPath + System.getProperty("file.separator") + "data.txt"; + String destFilename = rootPath + OSUtils.getFileSeparator() + "data.txt"; logger.info("destFilename: "+destFilename); //Define test resource diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 4320bd9710..a9d1c532fb 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -1024,11 +1024,6 @@ public final class Constants { public static final String END_TIME = "end time"; public static final String START_END_DATE = "startDate,endDate"; - /** - * system line separator - */ - public static final String SYSTEM_LINE_SEPARATOR = System.getProperty("line.separator"); - /** * net system properties */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/DolphinPluginManagerConfig.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/DolphinPluginManagerConfig.java index 7979868ef4..68e3da4f7b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/DolphinPluginManagerConfig.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/DolphinPluginManagerConfig.java @@ -20,6 +20,8 @@ package org.apache.dolphinscheduler.common.plugin; import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import org.apache.dolphinscheduler.common.utils.OSUtils; + import java.io.File; import java.util.List; @@ -46,7 +48,7 @@ public class DolphinPluginManagerConfig { /** * Development, When AlertServer is running on IDE, AlertPluginLoad can load Alert Plugin from local Repository. */ - private String mavenLocalRepository = System.getProperty("user.home") + "/.m2/repository"; + private String mavenLocalRepository = OSUtils.getUserHome() + "/.m2/repository"; private List mavenRemoteRepository = ImmutableList.of("http://repo1.maven.org/maven2/"); File getInstalledPluginsDir() { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java index 9a24fdb407..ba01deb15f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.common.shell; +import org.apache.dolphinscheduler.common.utils.OSUtils; + import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -169,7 +171,7 @@ public abstract class AbstractShell { String line = errReader.readLine(); while((line != null) && !isInterrupted()) { errMsg.append(line); - errMsg.append(System.getProperty("line.separator")); + errMsg.append(OSUtils.getLineSeparator()); line = errReader.readLine(); } } catch(IOException ioe) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java index 4f0953f824..c2777203bf 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java @@ -260,7 +260,7 @@ public class NetUtils { } private static boolean isSpecifyNetworkInterface(NetworkInterface networkInterface) { - String preferredNetworkInterface = System.getProperty(DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE); + String preferredNetworkInterface = PropertyUtils.getString(DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE); return Objects.equals(networkInterface.getDisplayName(), preferredNetworkInterface); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index 3851f77e94..c96ef9e25e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -381,7 +381,7 @@ public class OSUtils { */ public static String getGroup() throws IOException { if (isWindows()) { - String currentProcUserName = System.getProperty("user.name"); + String currentProcUserName = getUserName(); String result = exeCmd(String.format("net user \"%s\"", currentProcUserName)); String line = result.split("\n")[22]; String group = PATTERN.split(line)[1]; @@ -479,6 +479,51 @@ public class OSUtils { return System.getProperty("os.name"); } + /** + * get current OS user name + * + * @return current OS user name + */ + public static String getUserName() { + return System.getProperty("user.name"); + } + + /** + * get current OS user home + * + * @return current OS user home + */ + public static String getUserHome() { + return System.getProperty("user.home"); + } + + /** + * get current User's working directory + * + * @return current User's working directory + */ + public static String getUserDir() { + return System.getProperty("user.dir"); + } + + /** + * get file separator + * + * @return file separator + */ + public static String getFileSeparator() { + return System.getProperty("file.separator"); + } + + /** + * get line separator + * + * @return line separator + */ + public static String getLineSeparator() { + return System.getProperty("line.separator"); + } + /** * check memory and cpu usage * diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 726c799c72..ab4e84d437 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -86,3 +86,9 @@ sudo.enable=true # development state development.state=false + +# allow ambiguous commands flag +jdk.lang.Process.allowAmbiguousCommands = true + +# preferred network interface for dolphin scheduler, if not, keep this value empty +dolphin.scheduler.network.interface.preferred = diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index f0ffc52f68..5633ea4e21 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -43,7 +43,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); private static final String T_VERSION_NAME = "t_escheduler_version"; private static final String T_NEW_VERSION_NAME = "t_ds_version"; - private static final String rootDir = System.getProperty("user.dir"); + private static final String rootDir = OSUtils.getUserDir(); protected static final DataSource dataSource = getDataSource(); private static final DbType dbType = getCurrentDbType(); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java index 9ac1a9a72f..c51d95e631 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java @@ -57,11 +57,6 @@ public class Constants { */ public static final String NETTY_EPOLL_ENABLE = System.getProperty("netty.epoll.enable", "true"); - /** - * OS Name - */ - public static final String OS_NAME = System.getProperty("os.name"); - /** * warm up time */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java index 89eb1f9607..06420eee10 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.remote.utils; +import org.apache.dolphinscheduler.common.utils.OSUtils; + import io.netty.channel.epoll.Epoll; import io.netty.channel.epoll.EpollServerSocketChannel; import io.netty.channel.epoll.EpollSocketChannel; @@ -34,7 +36,7 @@ public class NettyUtils { } public static boolean useEpoll() { - String osName = Constants.OS_NAME; + String osName = OSUtils.getOSName(); if (!osName.toLowerCase().contains("linux")) { return false; } diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/NettyUtilTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/NettyUtilTest.java index a3e13850ac..879449eb86 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/NettyUtilTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/NettyUtilTest.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.remote.utils; -import static org.apache.dolphinscheduler.remote.utils.Constants.OS_NAME; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.junit.Assert; import org.junit.Test; @@ -31,7 +31,7 @@ public class NettyUtilTest { @Test public void testUserEpoll() { - if (OS_NAME.toLowerCase().contains("linux") && Epoll.isAvailable()) { + if (OSUtils.getOSName().toLowerCase().contains("linux") && Epoll.isAvailable()) { Assert.assertTrue(NettyUtils.useEpoll()); } else { Assert.assertFalse(NettyUtils.useEpoll()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index 4d2ace663e..42ab37e7c4 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -123,7 +123,7 @@ public class ProcessUtils { boolean allowAmbiguousCommands = false; if (security == null) { allowAmbiguousCommands = true; - String value = System.getProperty(LOCAL_PROCESS_EXEC); + String value = PropertyUtils.getString(LOCAL_PROCESS_EXEC); if (value != null) { allowAmbiguousCommands = !Constants.STRING_FALSE.equalsIgnoreCase(value); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java index f4e9080a68..5d95860e27 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.server.worker.task; +import org.apache.dolphinscheduler.common.utils.OSUtils; + import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -28,7 +30,7 @@ public class PythonCommandExecutorTest { @Test public void testGetPythonHome() { - String path = System.getProperty("user.dir") + "/script/env/dolphinscheduler_env.sh"; + String path = OSUtils.getUserDir() + "/script/env/dolphinscheduler_env.sh"; if (path.contains("dolphinscheduler-server/")) { path = path.replace("dolphinscheduler-server/", ""); } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java index 7ac23a3c4d..b73e774be3 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.service.zk; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.service.exceptions.ServiceException; @@ -117,7 +118,7 @@ public class ZKServer { * @param port The port to listen on */ public void startLocalZkServer(final int port) { - String zkDataDir = System.getProperty("user.dir") + (StringUtils.isEmpty(prefix) ? StringUtils.EMPTY : ("/" + prefix)) + "/zookeeper_data"; + String zkDataDir = OSUtils.getUserDir() + (StringUtils.isEmpty(prefix) ? StringUtils.EMPTY : ("/" + prefix)) + "/zookeeper_data"; File file = new File(zkDataDir); if (file.exists()) { logger.warn("The path of zk server exists"); -- Gitee