Aaron Wang
1 year ago
committed by
GitHub
6 changed files with 232 additions and 0 deletions
@ -0,0 +1,31 @@
|
||||
#!/bin/bash |
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
BIN_DIR=$(dirname $0) |
||||
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} |
||||
|
||||
if [ "$DOCKER" != "true" ]; then |
||||
source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" |
||||
fi |
||||
|
||||
JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} |
||||
|
||||
$JAVA_HOME/bin/java $JAVA_OPTS \ |
||||
-cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \ |
||||
-Dspring.profiles.active=resource,${DATABASE} \ |
||||
org.apache.dolphinscheduler.tools.resource.MigrateResource $1 |
@ -0,0 +1,55 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.tools.resource; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.boot.CommandLineRunner; |
||||
import org.springframework.boot.SpringApplication; |
||||
import org.springframework.boot.autoconfigure.SpringBootApplication; |
||||
import org.springframework.context.annotation.ComponentScan; |
||||
import org.springframework.context.annotation.Profile; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
@SpringBootApplication |
||||
@ComponentScan("org.apache.dolphinscheduler") |
||||
public class MigrateResource { |
||||
|
||||
public static void main(String[] args) { |
||||
SpringApplication.run(MigrateResource.class, args); |
||||
} |
||||
|
||||
@Component |
||||
@Profile("resource") |
||||
static class MigrateResourceRunner implements CommandLineRunner { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(MigrateResourceRunner.class); |
||||
|
||||
@Autowired |
||||
private MigrateResourceService migrateResourceService; |
||||
|
||||
@Override |
||||
public void run(String... args) { |
||||
String targetTenantCode = args[0]; |
||||
logger.info("Moving all unmanaged resources to tenant: {}", targetTenantCode); |
||||
migrateResourceService.migrateResourceOnce(targetTenantCode); |
||||
} |
||||
} |
||||
|
||||
} |
@ -0,0 +1,110 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.tools.resource; |
||||
|
||||
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; |
||||
|
||||
import org.apache.dolphinscheduler.dao.entity.Resource; |
||||
import org.apache.dolphinscheduler.dao.entity.UdfFunc; |
||||
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.TenantMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; |
||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; |
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType; |
||||
|
||||
import org.apache.zookeeper.common.StringUtils; |
||||
|
||||
import java.io.IOException; |
||||
import java.util.List; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Service; |
||||
|
||||
@Service |
||||
public class MigrateResourceService { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(MigrateResourceService.class); |
||||
|
||||
@Autowired |
||||
private StorageOperate storageOperate; |
||||
|
||||
@Autowired |
||||
private TenantMapper tenantMapper; |
||||
|
||||
@Autowired |
||||
private ResourceMapper resourceMapper; |
||||
|
||||
@Autowired |
||||
private UdfFuncMapper udfFuncMapper; |
||||
|
||||
private static final String MIGRATE_BASE_DIR = ".migrate"; |
||||
|
||||
public void migrateResourceOnce(String targetTenantCode) { |
||||
if (true != tenantMapper.existTenant(targetTenantCode)) { |
||||
logger.error("Tenant not exists!"); |
||||
return; |
||||
} |
||||
|
||||
String resMigrateBasePath = createMigrateDirByType(targetTenantCode, ResourceType.FILE); |
||||
String udfMigrateBasePath = createMigrateDirByType(targetTenantCode, ResourceType.UDF); |
||||
if (StringUtils.isEmpty(resMigrateBasePath) || StringUtils.isEmpty(udfMigrateBasePath)) { |
||||
return; |
||||
} |
||||
|
||||
// migrate all unmanaged resources and udfs once
|
||||
List<Resource> resources = resourceMapper.queryResourceExceptUserId(-1); |
||||
resources.forEach(resource -> { |
||||
try { |
||||
String oriFullName = resource.getFullName(); |
||||
oriFullName = oriFullName.startsWith("/") ? oriFullName.substring(1) : oriFullName; |
||||
if (resource.getType().equals(ResourceType.FILE)) { |
||||
storageOperate.copy(oriFullName, |
||||
String.format(FORMAT_S_S, resMigrateBasePath, oriFullName), true, true); |
||||
} else if (resource.getType().equals(ResourceType.UDF)) { |
||||
String fullName = String.format(FORMAT_S_S, udfMigrateBasePath, oriFullName); |
||||
storageOperate.copy(oriFullName, fullName, true, true); |
||||
|
||||
// change relative udfs resourceName
|
||||
List<UdfFunc> udfs = udfFuncMapper.listUdfByResourceId(new Integer[]{resource.getId()}); |
||||
udfs.forEach(udf -> { |
||||
udf.setResourceName(fullName); |
||||
udfFuncMapper.updateById(udf); |
||||
}); |
||||
} |
||||
} catch (IOException e) { |
||||
logger.error("Migrate resource failed: {}", e.getMessage()); |
||||
} |
||||
}); |
||||
} |
||||
|
||||
public String createMigrateDirByType(String targetTenantCode, ResourceType type) { |
||||
String migrateBasePath = type.equals(ResourceType.FILE) ? storageOperate.getResDir(targetTenantCode) |
||||
: storageOperate.getUdfDir(targetTenantCode); |
||||
migrateBasePath += MIGRATE_BASE_DIR; |
||||
try { |
||||
storageOperate.mkdir(targetTenantCode, migrateBasePath); |
||||
} catch (IOException e) { |
||||
logger.error("create migrate base directory {} failed", migrateBasePath); |
||||
return ""; |
||||
} |
||||
return migrateBasePath; |
||||
} |
||||
|
||||
} |
Loading…
Reference in new issue