Browse Source

[2.0.0-chery-pick #7003]Fix shell tools for database schema cannot run (#7087)

PR #7003
ISSUE #6998
2.0.7-release
Kirs 3 years ago committed by GitHub
parent
commit
b3e8704194
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 42
      docker/build/hooks/build
  2. 2
      docker/build/startup-init-conf.sh
  3. 117
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java
  4. 155
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
  5. 118
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java
  6. 5
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
  7. 1
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java
  8. 60
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java
  9. 6
      script/create-dolphinscheduler.sh
  10. 16
      script/dolphinscheduler-daemon.sh
  11. 2
      script/scp-hosts.sh
  12. 6
      script/upgrade-dolphinscheduler.sh

42
docker/build/hooks/build

@ -18,41 +18,17 @@
set -e set -e
echo "------ dolphinscheduler start - build -------" ROOT_DIR=$(dirname "$0")/../../..
printenv MVN="$ROOT_DIR"/mvnw
VERSION=$("$MVN" -q -DforceStdout -N org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version)
if [ -z "${VERSION}" ] DOCKER_REPO=${DOCKER_REPO:-"apache/dolphinscheduler"}
then TAG=${TAG:-"$VERSION"}
echo "set default environment variable [VERSION]"
export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
fi
if [ "${DOCKER_REPO}x" = "x" ] echo "Building Docker image as: $DOCKER_REPO:$TAG"
then
echo "set default environment variable [DOCKER_REPO]"
export DOCKER_REPO='apache/dolphinscheduler'
fi
echo "Version: $VERSION" "$MVN" -B clean package -Prelease -Dmaven.test.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
echo "Repo: $DOCKER_REPO"
echo -e "Current Directory is $(pwd)\n" cp "$ROOT_DIR"/dolphinscheduler-dist/target/apache-dolphinscheduler-$VERSION-bin.tar.gz "$ROOT_DIR"/docker/build/
# maven package(Project Directory) docker build --build-arg VERSION=$VERSION -t $DOCKER_REPO:$TAG "$ROOT_DIR"/docker/build/
echo -e "./mvnw -B clean package -Prelease -Dmaven.test.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120"
./mvnw -B clean package -Prelease -Dmaven.test.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
# mv dolphinscheduler-bin.tar.gz file to docker/build directory
echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSION}-bin.tar.gz $(pwd)/docker/build/\n"
mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSION}-bin.tar.gz $(pwd)/docker/build/
# docker build
BUILD_COMMAND="docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/"
echo -e "$BUILD_COMMAND\n"
if (docker info 2> /dev/null | grep -i "ERROR"); then
sudo $BUILD_COMMAND
else
$BUILD_COMMAND
fi
echo "------ dolphinscheduler end - build -------"

2
docker/build/startup-init-conf.sh

@ -37,6 +37,8 @@ echo "init env variables"
[ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_URL="jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS}" [ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_URL="jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS}"
[ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_USERNAME=${DATABASE_USERNAME} [ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_USERNAME=${DATABASE_USERNAME}
[ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_PASSWORD=${DATABASE_PASSWORD} [ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_PASSWORD=${DATABASE_PASSWORD}
# export SPRING_DATASOURCE_DRIVER_CLASS_NAME=${DATABASE_DRIVER}
#============================================================================ #============================================================================
# Registry # Registry

117
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java

@ -1,117 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.Arrays;
import java.util.List;
@RunWith(PowerMockRunner.class)
@PrepareForTest({ LoggerFactory.class, FileUtils.class })
public class SchemaUtilsTest {
@Test
public void testReplaceBlank() {
Assert.assertEquals("abc", SchemaUtils.replaceBlank(" abc"));
Assert.assertEquals("abc", SchemaUtils.replaceBlank("abc "));
Assert.assertEquals("abc", SchemaUtils.replaceBlank("a b c"));
Assert.assertEquals("abc", SchemaUtils.replaceBlank("a b c"));
Assert.assertEquals("", SchemaUtils.replaceBlank(" "));
Assert.assertEquals("", SchemaUtils.replaceBlank(null));
Assert.assertEquals("我怕的你", SchemaUtils.replaceBlank("我怕的 你"));
}
@Test
public void testGetSoftVersion() {
// file not found
try {
SchemaUtils.getSoftVersion();
} catch (RuntimeException e) {
Assert.assertEquals("Failed to get the product version description file. The file could not be found",
e.getMessage());
}
// file exists, fmt is invalid
FileUtils.writeContent2File("32432423", "sql/soft_version");
Assert.assertEquals("32432423", SchemaUtils.getSoftVersion());
}
@Test
public void testIsAGreatVersion() {
// param is null
try {
SchemaUtils.isAGreatVersion(null, null);
} catch (RuntimeException e) {
Assert.assertEquals("schemaVersion or version is empty", e.getMessage());
}
// param is ""
try {
SchemaUtils.isAGreatVersion("", "");
} catch (RuntimeException e) {
Assert.assertEquals("schemaVersion or version is empty", e.getMessage());
}
Assert.assertFalse(SchemaUtils.isAGreatVersion("1", "1"));
Assert.assertTrue(SchemaUtils.isAGreatVersion("2", "1"));
Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1"));
Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1.0.1"));
Assert.assertFalse(SchemaUtils.isAGreatVersion("1.1", "1.2"));
Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1.1", "1.1"));
Assert.assertTrue(SchemaUtils.isAGreatVersion("10.1.1", "1.01.100"));
try {
SchemaUtils.isAGreatVersion("10.1.1", ".1");
} catch (Exception e) {
Assert.assertNotNull(e);
}
try {
SchemaUtils.isAGreatVersion("a.1.1", "b.1");
} catch (Exception e) {
Assert.assertNotNull(e);
}
}
@Test
public void testGetAllSchemaList() {
//normal
PowerMockito.mockStatic(FileUtils.class);
File[] files = new File[4];
files[0] = new File("sql/upgrade/1.2.0_schema");
files[1] = new File("sql/upgrade/1.0.1_schema");
files[2] = new File("sql/upgrade/1.0.2_schema");
files[3] = new File("sql/upgrade/1.1.0_schema");
PowerMockito.when(FileUtils.getAllDir("sql/upgrade")).thenReturn(files);
List<String> real = SchemaUtils.getAllSchemaList();
List<String> expect = Arrays.asList("1.0.1_schema", "1.0.2_schema",
"1.1.0_schema", "1.2.0_schema");
boolean result = org.apache.commons.collections.CollectionUtils.isEqualCollection(real, expect);
Assert.assertTrue(result);
//normal
files = new File[0];
PowerMockito.when(FileUtils.getAllDir("sql/upgrade")).thenReturn(files);
real = SchemaUtils.getAllSchemaList();
Assert.assertNull(real);
}
}

155
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java

@ -17,160 +17,38 @@
package org.apache.dolphinscheduler.dao.datasource; package org.apache.dolphinscheduler.dao.datasource;
import static org.apache.dolphinscheduler.common.Constants.DATASOURCE_PROPERTIES; import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import org.apache.dolphinscheduler.common.Constants; import com.baomidou.mybatisplus.core.config.GlobalConfig;
import org.apache.dolphinscheduler.common.enums.ProfileType; import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor;
import org.apache.dolphinscheduler.common.utils.PropertyUtils; import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean;
import org.apache.commons.lang.StringUtils;
import org.apache.ibatis.mapping.DatabaseIdProvider; import org.apache.ibatis.mapping.DatabaseIdProvider;
import org.apache.ibatis.mapping.VendorDatabaseIdProvider; import org.apache.ibatis.mapping.VendorDatabaseIdProvider;
import org.apache.ibatis.session.SqlSession; import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory; import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.type.JdbcType; import org.apache.ibatis.type.JdbcType;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.sql.DataSource;
import org.h2.Driver;
import org.mybatis.spring.SqlSessionTemplate; import org.mybatis.spring.SqlSessionTemplate;
import org.mybatis.spring.annotation.MapperScan;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.Environment;
import org.springframework.core.io.FileSystemResourceLoader;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.core.io.support.ResourcePatternResolver; import org.springframework.core.io.support.ResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType;
import com.baomidou.mybatisplus.annotation.IdType; import javax.sql.DataSource;
import com.baomidou.mybatisplus.core.MybatisConfiguration; import java.util.Properties;
import com.baomidou.mybatisplus.core.config.GlobalConfig;
import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor;
import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean;
import com.google.common.collect.Lists;
import com.zaxxer.hikari.HikariDataSource;
/**
* data source connection factory
*/
@Configuration @Configuration
@MapperScan("org.apache.dolphinscheduler.*.mapper")
public class SpringConnectionFactory { public class SpringConnectionFactory {
private static final Logger logger = LoggerFactory.getLogger(SpringConnectionFactory.class);
@Autowired
private Environment environment;
private static final AtomicBoolean H2_INITIALIZED = new AtomicBoolean(false);
@PostConstruct
public void init() {
String datasourceProfile = getSpringActiveProfile().stream()
.filter(ProfileType.DATASOURCE_PROFILE::contains)
.findFirst()
.orElse("");
if (StringUtils.isEmpty(datasourceProfile) || ProfileType.MYSQL.equals(datasourceProfile)) {
// default load datasource.properties
PropertyUtils.loadPropertyFile(DATASOURCE_PROPERTIES.replace("-%s", ""));
} else {
// load datasource-{spring.profiles.active}.properties
PropertyUtils.loadPropertyFile(String.format(DATASOURCE_PROPERTIES, datasourceProfile));
}
}
/**
* pagination interceptor
*
* @return pagination interceptor
*/
@Bean @Bean
public PaginationInterceptor paginationInterceptor() { public PaginationInterceptor paginationInterceptor() {
return new PaginationInterceptor(); return new PaginationInterceptor();
} }
/**
* get the data source
*
* @return dataSource
*/
@Bean(destroyMethod = "", name = "datasource")
public DataSource dataSource() throws SQLException {
String driverClassName = PropertyUtils.getString(Constants.SPRING_DATASOURCE_DRIVER_CLASS_NAME);
if (Driver.class.getName().equals(driverClassName)) {
initializeH2Datasource();
}
HikariDataSource dataSource = new HikariDataSource();
dataSource.setDriverClassName(driverClassName);
dataSource.setJdbcUrl(PropertyUtils.getString(Constants.SPRING_DATASOURCE_URL));
dataSource.setUsername(PropertyUtils.getString(Constants.SPRING_DATASOURCE_USERNAME));
dataSource.setPassword(PropertyUtils.getString(Constants.SPRING_DATASOURCE_PASSWORD));
dataSource.setConnectionTestQuery(PropertyUtils.getString(Constants.SPRING_DATASOURCE_VALIDATION_QUERY, "SELECT 1"));
dataSource.setMinimumIdle(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MIN_IDLE, 5));
dataSource.setMaximumPoolSize(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE, 50));
dataSource.setConnectionTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_CONNECTION_TIMEOUT, 30000));
dataSource.setIdleTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_IDLE_TIMEOUT, 600000));
dataSource.setMaxLifetime(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_LIFE_TIME, 1800000));
dataSource.setValidationTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_VALIDATION_TIMEOUT, 5000));
dataSource.setLeakDetectionThreshold(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_LEAK_DETECTION_THRESHOLD, 0));
dataSource.setInitializationFailTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_INITIALIZATION_FAIL_TIMEOUT, 1));
dataSource.setAutoCommit(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_IS_AUTOCOMMIT, true));
dataSource.addDataSourceProperty(Constants.CACHE_PREP_STMTS, PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_CACHE_PREP_STMTS, true));
dataSource.addDataSourceProperty(Constants.PREP_STMT_CACHE_SIZE, PropertyUtils.getInt(Constants.SPRING_DATASOURCE_PREP_STMT_CACHE_SIZE, 250));
dataSource.addDataSourceProperty(Constants.PREP_STMT_CACHE_SQL_LIMIT, PropertyUtils.getInt(Constants.SPRING_DATASOURCE_PREP_STMT_CACHE_SQL_LIMIT, 2048));
logger.info("Initialize DataSource DataSource success");
return dataSource;
}
private void initializeH2Datasource() {
if (H2_INITIALIZED.compareAndSet(false, true)) {
EmbeddedDatabaseBuilder embeddedDatabaseBuilder = new EmbeddedDatabaseBuilder(new FileSystemResourceLoader());
embeddedDatabaseBuilder
.setType(EmbeddedDatabaseType.H2)
.setScriptEncoding(Constants.UTF_8)
.setName("dolphinscheduler;MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1")
.addScript(PropertyUtils.getString("spring.datasource.sql.schema", "classpath:sql/dolphinscheduler_h2.sql"))
.build();
logger.info("Initialize H2 DataSource success");
}
}
/**
* * get transaction manager
*
* @return DataSourceTransactionManager
*/
@Bean @Bean
public DataSourceTransactionManager transactionManager(DataSource dataSource) { public DataSourceTransactionManager transactionManager(DataSource dataSource) {
return new DataSourceTransactionManager(dataSource); return new DataSourceTransactionManager(dataSource);
} }
/**
* * get sql session factory
*
* @return sqlSessionFactory
* @throws Exception sqlSessionFactory exception
*/
@Bean @Bean
public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Exception { public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Exception {
MybatisConfiguration configuration = new MybatisConfiguration(); MybatisConfiguration configuration = new MybatisConfiguration();
@ -179,6 +57,7 @@ public class SpringConnectionFactory {
configuration.setCallSettersOnNulls(true); configuration.setCallSettersOnNulls(true);
configuration.setJdbcTypeForNull(JdbcType.NULL); configuration.setJdbcTypeForNull(JdbcType.NULL);
configuration.addInterceptor(paginationInterceptor()); configuration.addInterceptor(paginationInterceptor());
configuration.setGlobalConfig(new GlobalConfig().setBanner(false));
MybatisSqlSessionFactoryBean sqlSessionFactoryBean = new MybatisSqlSessionFactoryBean(); MybatisSqlSessionFactoryBean sqlSessionFactoryBean = new MybatisSqlSessionFactoryBean();
sqlSessionFactoryBean.setConfiguration(configuration); sqlSessionFactoryBean.setConfiguration(configuration);
sqlSessionFactoryBean.setDataSource(dataSource); sqlSessionFactoryBean.setDataSource(dataSource);
@ -196,11 +75,6 @@ public class SpringConnectionFactory {
return sqlSessionFactoryBean.getObject(); return sqlSessionFactoryBean.getObject();
} }
/**
* get sql session
*
* @return SqlSession
*/
@Bean @Bean
public SqlSession sqlSession(SqlSessionFactory sqlSessionFactory) { public SqlSession sqlSession(SqlSessionFactory sqlSessionFactory) {
return new SqlSessionTemplate(sqlSessionFactory); return new SqlSessionTemplate(sqlSessionFactory);
@ -216,17 +90,4 @@ public class SpringConnectionFactory {
databaseIdProvider.setProperties(properties); databaseIdProvider.setProperties(properties);
return databaseIdProvider; return databaseIdProvider;
} }
/**
* Get spring active profile, which will be set by -Dspring.profiles.active=api or in application.xml
*
* @return
*/
private List<String> getSpringActiveProfile() {
if (environment != null) {
return Lists.newArrayList(environment.getActiveProfiles());
}
String property = System.getProperty("spring.profiles.active", "");
return Arrays.stream(property.split(",")).map(String::trim).collect(Collectors.toList());
}
} }

118
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java

@ -0,0 +1,118 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
import com.google.common.base.Strings;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.io.ClassPathResource;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
/**
* Metadata related common classes
*/
public class SchemaUtils {
private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class);
private SchemaUtils() {
throw new UnsupportedOperationException("Construct SchemaUtils");
}
public static List<String> getAllSchemaList() throws IOException {
final File[] schemaDirArr = new ClassPathResource("sql/upgrade").getFile().listFiles();
if (schemaDirArr == null || schemaDirArr.length == 0) {
return Collections.emptyList();
}
return Arrays.stream(schemaDirArr).map(File::getName).sorted((o1, o2) -> {
try {
String version1 = o1.split("_")[0];
String version2 = o2.split("_")[0];
if (version1.equals(version2)) {
return 0;
}
if (SchemaUtils.isAGreatVersion(version1, version2)) {
return 1;
}
return -1;
} catch (Exception e) {
logger.error(e.getMessage(), e);
throw new RuntimeException(e);
}
}).collect(Collectors.toList());
}
/**
* Determine whether schemaVersion is higher than version
*
* @param schemaVersion schema version
* @param version version
* @return Determine whether schemaVersion is higher than version
*/
public static boolean isAGreatVersion(String schemaVersion, String version) {
if (StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) {
throw new RuntimeException("schemaVersion or version is empty");
}
String[] schemaVersionArr = schemaVersion.split("\\.");
String[] versionArr = version.split("\\.");
int arrLength = Math.min(schemaVersionArr.length, versionArr.length);
for (int i = 0; i < arrLength; i++) {
if (Integer.parseInt(schemaVersionArr[i]) > Integer.parseInt(versionArr[i])) {
return true;
} else if (Integer.parseInt(schemaVersionArr[i]) < Integer.parseInt(versionArr[i])) {
return false;
}
}
// If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number
return schemaVersionArr.length > versionArr.length;
}
/**
* Gets the current software version number of the system
*
* @return current software version
*/
public static String getSoftVersion() throws IOException {
final ClassPathResource softVersionFile = new ClassPathResource("sql/soft_version");
String softVersion;
try {
softVersion = FileUtils.readFile2Str(softVersionFile.getInputStream());
softVersion = Strings.nullToEmpty(softVersion).replaceAll("\\s+|\r|\n", "");
} catch (FileNotFoundException e) {
logger.error(e.getMessage(), e);
throw new RuntimeException("Failed to get the product version description file. The file could not be found", e);
}
return softVersion;
}
}

5
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

@ -28,7 +28,10 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ScriptRunner;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;

1
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java

@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.dao; package org.apache.dolphinscheduler.dao;
import org.apache.dolphinscheduler.common.enums.ProfileType; import org.apache.dolphinscheduler.common.enums.ProfileType;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback; import org.springframework.test.annotation.Rollback;

60
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java

@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
import org.junit.Assert;
import org.junit.Test;
public class SchemaUtilsTest {
@Test
public void testIsAGreatVersion() {
// param is null
try {
SchemaUtils.isAGreatVersion(null, null);
} catch (RuntimeException e) {
Assert.assertEquals("schemaVersion or version is empty", e.getMessage());
}
// param is ""
try {
SchemaUtils.isAGreatVersion("", "");
} catch (RuntimeException e) {
Assert.assertEquals("schemaVersion or version is empty", e.getMessage());
}
Assert.assertFalse(SchemaUtils.isAGreatVersion("1", "1"));
Assert.assertTrue(SchemaUtils.isAGreatVersion("2", "1"));
Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1"));
Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1.0.1"));
Assert.assertFalse(SchemaUtils.isAGreatVersion("1.1", "1.2"));
Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1.1", "1.1"));
Assert.assertTrue(SchemaUtils.isAGreatVersion("10.1.1", "1.01.100"));
try {
SchemaUtils.isAGreatVersion("10.1.1", ".1");
Assert.fail("Should fail");
} catch (Exception ignored) {
// This is expected
}
try {
SchemaUtils.isAGreatVersion("a.1.1", "b.1");
Assert.fail("Should fail");
} catch (Exception ignored) {
// This is expected
}
}
}

6
script/create-dolphinscheduler.sh

@ -22,16 +22,20 @@ DOLPHINSCHEDULER_HOME=$BIN_DIR/..
export JAVA_HOME=$JAVA_HOME export JAVA_HOME=$JAVA_HOME
export DATABASE_TYPE=${DATABASE_TYPE:-"h2"}
export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"}
export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},${DATABASE_TYPE}"
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
export DOLPHINSCHEDULER_SQL_DIR=$DOLPHINSCHEDULER_HOME/sql
export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
export STOP_TIMEOUT=5 export STOP_TIMEOUT=5
CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler
exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_SQL_DIR:$DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
cd $DOLPHINSCHEDULER_HOME cd $DOLPHINSCHEDULER_HOME
$JAVA_HOME/bin/java $exec_command $JAVA_HOME/bin/java $exec_command

16
script/dolphinscheduler-daemon.sh

@ -38,8 +38,6 @@ DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd`
source /etc/profile source /etc/profile
source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh" source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh"
export JAVA_HOME=$JAVA_HOME
#export JAVA_HOME=/opt/soft/jdk
export HOSTNAME=`hostname` export HOSTNAME=`hostname`
export DOLPHINSCHEDULER_PID_DIR=$DOLPHINSCHEDULER_HOME/pid export DOLPHINSCHEDULER_PID_DIR=$DOLPHINSCHEDULER_HOME/pid
@ -61,36 +59,40 @@ cd $DOLPHINSCHEDULER_HOME
export DOLPHINSCHEDULER_OPTS="-server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm $DOLPHINSCHEDULER_OPTS" export DOLPHINSCHEDULER_OPTS="-server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm $DOLPHINSCHEDULER_OPTS"
export DATABASE_TYPE=${DATABASE_TYPE:-"h2"}
export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"}
if [ "$command" = "api-server" ]; then if [ "$command" = "api-server" ]; then
LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api" LOG_FILE="-Dlogging.config=classpath:logback-api.xml"
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m" HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $API_SERVER_OPTS" export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $API_SERVER_OPTS"
export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},api,${DATABASE_TYPE}"
elif [ "$command" = "master-server" ]; then elif [ "$command" = "master-server" ]; then
LOG_FILE="-Dlogging.config=classpath:logback-master.xml" LOG_FILE="-Dlogging.config=classpath:logback-master.xml"
CLASS=org.apache.dolphinscheduler.server.master.MasterServer CLASS=org.apache.dolphinscheduler.server.master.MasterServer
HEAP_OPTS="-Xms4g -Xmx4g -Xmn2g" HEAP_OPTS="-Xms4g -Xmx4g -Xmn2g"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $MASTER_SERVER_OPTS" export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $MASTER_SERVER_OPTS"
export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},master,${DATABASE_TYPE}"
elif [ "$command" = "worker-server" ]; then elif [ "$command" = "worker-server" ]; then
LOG_FILE="-Dlogging.config=classpath:logback-worker.xml" LOG_FILE="-Dlogging.config=classpath:logback-worker.xml"
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
HEAP_OPTS="-Xms2g -Xmx2g -Xmn1g" HEAP_OPTS="-Xms2g -Xmx2g -Xmn1g"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $WORKER_SERVER_OPTS" export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $WORKER_SERVER_OPTS"
export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},worker,${DATABASE_TYPE}"
elif [ "$command" = "alert-server" ]; then elif [ "$command" = "alert-server" ]; then
LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml" LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml"
CLASS=org.apache.dolphinscheduler.alert.AlertServer CLASS=org.apache.dolphinscheduler.alert.AlertServer
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m" HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $ALERT_SERVER_OPTS" export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $ALERT_SERVER_OPTS"
export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},alert,${DATABASE_TYPE}"
elif [ "$command" = "logger-server" ]; then elif [ "$command" = "logger-server" ]; then
CLASS=org.apache.dolphinscheduler.server.log.LoggerServer CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m" HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $LOGGER_SERVER_OPTS" export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $LOGGER_SERVER_OPTS"
elif [ "$command" = "zookeeper-server" ]; then
#note: this command just for getting a quick experience,not recommended for production. this operation will start a standalone zookeeper server
LOG_FILE="-Dlogback.configurationFile=classpath:logback-zookeeper.xml"
CLASS=org.apache.dolphinscheduler.service.zk.ZKServer
elif [ "$command" = "standalone-server" ]; then elif [ "$command" = "standalone-server" ]; then
CLASS=org.apache.dolphinscheduler.server.StandaloneServer CLASS=org.apache.dolphinscheduler.server.StandaloneServer
export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},standalone,${DATABASE_TYPE}"
else else
echo "Error: No command named '$command' was found." echo "Error: No command named '$command' was found."
exit 1 exit 1

2
script/scp-hosts.sh

@ -29,6 +29,7 @@ fi
declare -A workersGroupMap=() declare -A workersGroupMap=()
workersGroup=(${workers//,/ }) workersGroup=(${workers//,/ })
# shellcheck disable=SC2068
for workerGroup in ${workersGroup[@]} for workerGroup in ${workersGroup[@]}
do do
echo $workerGroup; echo $workerGroup;
@ -39,6 +40,7 @@ done
hostsArr=(${ips//,/ }) hostsArr=(${ips//,/ })
# shellcheck disable=SC2068
for host in ${hostsArr[@]} for host in ${hostsArr[@]}
do do

6
script/upgrade-dolphinscheduler.sh

@ -22,16 +22,20 @@ DOLPHINSCHEDULER_HOME=$BIN_DIR/..
export JAVA_HOME=$JAVA_HOME export JAVA_HOME=$JAVA_HOME
export DATABASE_TYPE=${DATABASE_TYPE:-"h2"}
export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"}
export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},${DATABASE_TYPE}"
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
export DOLPHINSCHEDULER_SQL_DIR=$DOLPHINSCHEDULER_HOME/sql
export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
export STOP_TIMEOUT=5 export STOP_TIMEOUT=5
CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler
exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_SQL_DIR:$DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
cd $DOLPHINSCHEDULER_HOME cd $DOLPHINSCHEDULER_HOME
$JAVA_HOME/bin/java $exec_command $JAVA_HOME/bin/java $exec_command

Loading…
Cancel
Save