From f3134c7ce7fffe775a6ea3ab74c296ece6b51545 Mon Sep 17 00:00:00 2001 From: Aaron Wang Date: Wed, 8 Feb 2023 10:20:08 +0800 Subject: [PATCH] [Feature][Resource Center] One-time migration script for unmanaged resources (#13512) --- docs/docs/en/guide/upgrade/upgrade.md | 16 +++ docs/docs/zh/guide/upgrade/upgrade.md | 16 +++ dolphinscheduler-tools/pom.xml | 4 + .../src/main/bin/migrate-resource.sh | 31 +++++ .../tools/resource/MigrateResource.java | 55 +++++++++ .../resource/MigrateResourceService.java | 110 ++++++++++++++++++ 6 files changed, 232 insertions(+) create mode 100644 dolphinscheduler-tools/src/main/bin/migrate-resource.sh create mode 100644 dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResource.java create mode 100644 dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java diff --git a/docs/docs/en/guide/upgrade/upgrade.md b/docs/docs/en/guide/upgrade/upgrade.md index 6b9f740ae2..b4935b29e5 100644 --- a/docs/docs/en/guide/upgrade/upgrade.md +++ b/docs/docs/en/guide/upgrade/upgrade.md @@ -38,6 +38,22 @@ jar package and add it to the `./tools/libs` directory, then export the followin Execute database upgrade script: `sh ./tools/bin/upgrade-schema.sh` +### Migrate Resource + +After refactoring resource center in version 3.2.0, original resources become unmanaged. You can assign a target tenant and execute one-time migration script. All resources will be migrated to directory `.migrate` of target tenant. + +#### Example + +Assign an existed target tenant `abc`, the base resource path is `/dolphinscheduler/abc/`. + +Execute script: `sh ./tools/bin/migrate-resource.sh abc`. + +Execution result: + +- The original file resource `a/b.sh` migrates to `/dolphinscheduler/abc/resources/.migrate/a/b.sh`. +- The original UDF resource `x/y.jar` migrates to `/dolphinscheduler/abc/udf/.migrate/x/y.jar`. +- Update UDF function's bound resource info. + ### Upgrade Service #### Change Configuration `bin/env/install_env.sh` diff --git a/docs/docs/zh/guide/upgrade/upgrade.md b/docs/docs/zh/guide/upgrade/upgrade.md index a9b15be585..1e2301c84f 100644 --- a/docs/docs/zh/guide/upgrade/upgrade.md +++ b/docs/docs/zh/guide/upgrade/upgrade.md @@ -37,6 +37,22 @@ jar 包 并添加到 `./tools/libs` 目录下，设置以下环境变量 执行数据库升级脚本：`sh ./tools/bin/upgrade-schema.sh` +### 资源迁移 + +3.2.0 版本资源中心重构，原资源中心内的资源将不受管理，您可以指定迁移到的目标租户，然后运行一次性资源迁移脚本，所有资源会迁移到目标租户的 .migrate 目录下。 + +#### 示例： + +指定已存在目标租户 `abc`，其资源根目录为 `/dolphinscheduler/abc/`。 + +执行脚本：`sh ./tools/bin/migrate-resource.sh abc`。 + +执行结果： + +- 原文件资源 `a/b.sh` 迁移至 `/dolphinscheduler/abc/resources/.migrate/a/b.sh`。 +- 原 UDF 资源 `x/y.jar` 迁移至 `/dolphinscheduler/abc/udf/.migrate/x/y.jar`。 +- 更新 UDF 函数绑定资源信息。 + ### 服务升级 #### 修改 `bin/env/install_env.sh` 配置内容 diff --git a/dolphinscheduler-tools/pom.xml b/dolphinscheduler-tools/pom.xml index d9494c0b09..e2a42529e0 100644 --- a/dolphinscheduler-tools/pom.xml +++ b/dolphinscheduler-tools/pom.xml @@ -46,6 +46,10 @@ org.apache.dolphinscheduler dolphinscheduler-dao + + org.apache.dolphinscheduler + dolphinscheduler-storage-all + org.apache.dolphinscheduler dolphinscheduler-aop diff --git a/dolphinscheduler-tools/src/main/bin/migrate-resource.sh b/dolphinscheduler-tools/src/main/bin/migrate-resource.sh new file mode 100644 index 0000000000..223a5d66b3 --- /dev/null +++ b/dolphinscheduler-tools/src/main/bin/migrate-resource.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BIN_DIR=$(dirname $0) +DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} + +if [ "$DOCKER" != "true" ]; then + source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" +fi + +JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} + +$JAVA_HOME/bin/java $JAVA_OPTS \ + -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \ + -Dspring.profiles.active=resource,${DATABASE} \ + org.apache.dolphinscheduler.tools.resource.MigrateResource $1 diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResource.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResource.java new file mode 100644 index 0000000000..e519dc05d7 --- /dev/null +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResource.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.tools.resource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@SpringBootApplication +@ComponentScan("org.apache.dolphinscheduler") +public class MigrateResource { + + public static void main(String[] args) { + SpringApplication.run(MigrateResource.class, args); + } + + @Component + @Profile("resource") + static class MigrateResourceRunner implements CommandLineRunner { + + private static final Logger logger = LoggerFactory.getLogger(MigrateResourceRunner.class); + + @Autowired + private MigrateResourceService migrateResourceService; + + @Override + public void run(String... args) { + String targetTenantCode = args[0]; + logger.info("Moving all unmanaged resources to tenant: {}", targetTenantCode); + migrateResourceService.migrateResourceOnce(targetTenantCode); + } + } + +} diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java new file mode 100644 index 0000000000..4f4076fc4b --- /dev/null +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.tools.resource; + +import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; + +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; +import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.TenantMapper; +import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import org.apache.zookeeper.common.StringUtils; + +import java.io.IOException; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class MigrateResourceService { + + private static final Logger logger = LoggerFactory.getLogger(MigrateResourceService.class); + + @Autowired + private StorageOperate storageOperate; + + @Autowired + private TenantMapper tenantMapper; + + @Autowired + private ResourceMapper resourceMapper; + + @Autowired + private UdfFuncMapper udfFuncMapper; + + private static final String MIGRATE_BASE_DIR = ".migrate"; + + public void migrateResourceOnce(String targetTenantCode) { + if (true != tenantMapper.existTenant(targetTenantCode)) { + logger.error("Tenant not exists!"); + return; + } + + String resMigrateBasePath = createMigrateDirByType(targetTenantCode, ResourceType.FILE); + String udfMigrateBasePath = createMigrateDirByType(targetTenantCode, ResourceType.UDF); + if (StringUtils.isEmpty(resMigrateBasePath) || StringUtils.isEmpty(udfMigrateBasePath)) { + return; + } + + // migrate all unmanaged resources and udfs once + List resources = resourceMapper.queryResourceExceptUserId(-1); + resources.forEach(resource -> { + try { + String oriFullName = resource.getFullName(); + oriFullName = oriFullName.startsWith("/") ? oriFullName.substring(1) : oriFullName; + if (resource.getType().equals(ResourceType.FILE)) { + storageOperate.copy(oriFullName, + String.format(FORMAT_S_S, resMigrateBasePath, oriFullName), true, true); + } else if (resource.getType().equals(ResourceType.UDF)) { + String fullName = String.format(FORMAT_S_S, udfMigrateBasePath, oriFullName); + storageOperate.copy(oriFullName, fullName, true, true); + + // change relative udfs resourceName + List udfs = udfFuncMapper.listUdfByResourceId(new Integer[]{resource.getId()}); + udfs.forEach(udf -> { + udf.setResourceName(fullName); + udfFuncMapper.updateById(udf); + }); + } + } catch (IOException e) { + logger.error("Migrate resource failed: {}", e.getMessage()); + } + }); + } + + public String createMigrateDirByType(String targetTenantCode, ResourceType type) { + String migrateBasePath = type.equals(ResourceType.FILE) ? storageOperate.getResDir(targetTenantCode) + : storageOperate.getUdfDir(targetTenantCode); + migrateBasePath += MIGRATE_BASE_DIR; + try { + storageOperate.mkdir(targetTenantCode, migrateBasePath); + } catch (IOException e) { + logger.error("create migrate base directory {} failed", migrateBasePath); + return ""; + } + return migrateBasePath; + } + +}