Aaron Wang
2 years ago
committed by
GitHub
6 changed files with 232 additions and 0 deletions
@ -0,0 +1,31 @@ |
|||||||
|
#!/bin/bash |
||||||
|
# |
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
# contributor license agreements. See the NOTICE file distributed with |
||||||
|
# this work for additional information regarding copyright ownership. |
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
# (the "License"); you may not use this file except in compliance with |
||||||
|
# the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, software |
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
# See the License for the specific language governing permissions and |
||||||
|
# limitations under the License. |
||||||
|
# |
||||||
|
|
||||||
|
BIN_DIR=$(dirname $0) |
||||||
|
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} |
||||||
|
|
||||||
|
if [ "$DOCKER" != "true" ]; then |
||||||
|
source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" |
||||||
|
fi |
||||||
|
|
||||||
|
JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} |
||||||
|
|
||||||
|
$JAVA_HOME/bin/java $JAVA_OPTS \ |
||||||
|
-cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \ |
||||||
|
-Dspring.profiles.active=resource,${DATABASE} \ |
||||||
|
org.apache.dolphinscheduler.tools.resource.MigrateResource $1 |
@ -0,0 +1,55 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.tools.resource; |
||||||
|
|
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||||
|
import org.springframework.boot.CommandLineRunner; |
||||||
|
import org.springframework.boot.SpringApplication; |
||||||
|
import org.springframework.boot.autoconfigure.SpringBootApplication; |
||||||
|
import org.springframework.context.annotation.ComponentScan; |
||||||
|
import org.springframework.context.annotation.Profile; |
||||||
|
import org.springframework.stereotype.Component; |
||||||
|
|
||||||
|
@SpringBootApplication |
||||||
|
@ComponentScan("org.apache.dolphinscheduler") |
||||||
|
public class MigrateResource { |
||||||
|
|
||||||
|
public static void main(String[] args) { |
||||||
|
SpringApplication.run(MigrateResource.class, args); |
||||||
|
} |
||||||
|
|
||||||
|
@Component |
||||||
|
@Profile("resource") |
||||||
|
static class MigrateResourceRunner implements CommandLineRunner { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(MigrateResourceRunner.class); |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private MigrateResourceService migrateResourceService; |
||||||
|
|
||||||
|
@Override |
||||||
|
public void run(String... args) { |
||||||
|
String targetTenantCode = args[0]; |
||||||
|
logger.info("Moving all unmanaged resources to tenant: {}", targetTenantCode); |
||||||
|
migrateResourceService.migrateResourceOnce(targetTenantCode); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,110 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.tools.resource; |
||||||
|
|
||||||
|
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.dao.entity.Resource; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.UdfFunc; |
||||||
|
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; |
||||||
|
import org.apache.dolphinscheduler.dao.mapper.TenantMapper; |
||||||
|
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; |
||||||
|
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; |
||||||
|
import org.apache.dolphinscheduler.spi.enums.ResourceType; |
||||||
|
|
||||||
|
import org.apache.zookeeper.common.StringUtils; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||||
|
import org.springframework.stereotype.Service; |
||||||
|
|
||||||
|
@Service |
||||||
|
public class MigrateResourceService { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(MigrateResourceService.class); |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private StorageOperate storageOperate; |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private TenantMapper tenantMapper; |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private ResourceMapper resourceMapper; |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private UdfFuncMapper udfFuncMapper; |
||||||
|
|
||||||
|
private static final String MIGRATE_BASE_DIR = ".migrate"; |
||||||
|
|
||||||
|
public void migrateResourceOnce(String targetTenantCode) { |
||||||
|
if (true != tenantMapper.existTenant(targetTenantCode)) { |
||||||
|
logger.error("Tenant not exists!"); |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
String resMigrateBasePath = createMigrateDirByType(targetTenantCode, ResourceType.FILE); |
||||||
|
String udfMigrateBasePath = createMigrateDirByType(targetTenantCode, ResourceType.UDF); |
||||||
|
if (StringUtils.isEmpty(resMigrateBasePath) || StringUtils.isEmpty(udfMigrateBasePath)) { |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
// migrate all unmanaged resources and udfs once
|
||||||
|
List<Resource> resources = resourceMapper.queryResourceExceptUserId(-1); |
||||||
|
resources.forEach(resource -> { |
||||||
|
try { |
||||||
|
String oriFullName = resource.getFullName(); |
||||||
|
oriFullName = oriFullName.startsWith("/") ? oriFullName.substring(1) : oriFullName; |
||||||
|
if (resource.getType().equals(ResourceType.FILE)) { |
||||||
|
storageOperate.copy(oriFullName, |
||||||
|
String.format(FORMAT_S_S, resMigrateBasePath, oriFullName), true, true); |
||||||
|
} else if (resource.getType().equals(ResourceType.UDF)) { |
||||||
|
String fullName = String.format(FORMAT_S_S, udfMigrateBasePath, oriFullName); |
||||||
|
storageOperate.copy(oriFullName, fullName, true, true); |
||||||
|
|
||||||
|
// change relative udfs resourceName
|
||||||
|
List<UdfFunc> udfs = udfFuncMapper.listUdfByResourceId(new Integer[]{resource.getId()}); |
||||||
|
udfs.forEach(udf -> { |
||||||
|
udf.setResourceName(fullName); |
||||||
|
udfFuncMapper.updateById(udf); |
||||||
|
}); |
||||||
|
} |
||||||
|
} catch (IOException e) { |
||||||
|
logger.error("Migrate resource failed: {}", e.getMessage()); |
||||||
|
} |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
public String createMigrateDirByType(String targetTenantCode, ResourceType type) { |
||||||
|
String migrateBasePath = type.equals(ResourceType.FILE) ? storageOperate.getResDir(targetTenantCode) |
||||||
|
: storageOperate.getUdfDir(targetTenantCode); |
||||||
|
migrateBasePath += MIGRATE_BASE_DIR; |
||||||
|
try { |
||||||
|
storageOperate.mkdir(targetTenantCode, migrateBasePath); |
||||||
|
} catch (IOException e) { |
||||||
|
logger.error("create migrate base directory {} failed", migrateBasePath); |
||||||
|
return ""; |
||||||
|
} |
||||||
|
return migrateBasePath; |
||||||
|
} |
||||||
|
|
||||||
|
} |
Loading…
Reference in new issue