Merge pull request 'KERNEL-5825 上传打包请求后端代码' (#1) from Jonas/plugin-bi-pack-request:master into master

Reviewed-on: #1

plugin.xml

@@ -1,4 +1,5 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?><plugin>
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<plugin>
     <id>com.fr.plugin.pack.request</id>
     <name><![CDATA[仪表板请求打包]]></name>
     <active>yes</active>
@@ -13,5 +14,11 @@
     <extra-decision>
         <WebResourceProvider class="com.fr.plugin.pack.ReportBridge"/>
         <WebResourceProvider class="com.fr.plugin.pack.ShowBridge"/>
+        <HttpHandlerProvider class="com.fr.plugin.pack.PackHandlerProvider"/>
+        <URLAliasProvider class="com.fr.plugin.pack.PackURLAliasProvider"/>
     </extra-decision>
+
+    <extra-core>
+        <WidgetExecuteProvider class="com.fr.plugin.pack.PackWidgetExecuteProvider"/>
+    </extra-core>
 </plugin>

pom.xml

@@ -9,7 +9,7 @@
         <version>5.0-RELEASE-SNAPSHOT</version>
         <relativePath>../nuclear-maven/pom.xml</relativePath>
     </parent>
-    
+
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>plugin-pack-request</artifactId>
@@ -20,6 +20,24 @@
             <artifactId>direct-adapter-sdk</artifactId>
             <version>${foundation-version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.finebi</groupId>
+            <artifactId>web-action</artifactId>
+            <version>5.0-RELEASE-SNAPSHOT</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.jmockit</groupId>
+            <artifactId>jmockit</artifactId>
+            <version>1.35</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.finebi</groupId>
+            <artifactId>web-test</artifactId>
+            <version>5.0-RELEASE-SNAPSHOT</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
@@ -34,7 +52,9 @@
                 </configuration>
             </plugin>
         </plugins>
-        <outputDirectory>/Users/leezerofly/env/direct-bi/WEB-INF/plugins/plugin-com.fr.plugin.pack.request-0.0.1/classes</outputDirectory>
+        <outputDirectory>
+            /Users/jonas/Desktop/代码/FineBI5.1/env/WebReport/WEB-INF/plugins/plugin-com.fr.plugin.pack.request-0.0.1/classes
+        </outputDirectory>
     </build>
 
 </project>

src/main/java/com/fr/plugin/pack/PackConstants.java

@@ -0,0 +1,14 @@
+package com.fr.plugin.pack;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-19
+ */
+public class PackConstants {
+    public final static String WIDGET_EXCEED_ERROR = "61310099";
+    public final static String REPORT_EXCEED_ERROR = "61310098";
+
+    public final static String WIDGET_TRAFFIC_NAME = "Widget";
+    public final static String REPORT_TRAFFIC_NAME = "Report";
+}

src/main/java/com/fr/plugin/pack/PackHandlerProvider.java

@@ -0,0 +1,19 @@
+package com.fr.plugin.pack;
+
+import com.fr.decision.fun.HttpHandler;
+import com.fr.decision.fun.impl.AbstractHttpHandlerProvider;
+import com.fr.plugin.pack.http.PackHttpHandler;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-19
+ */
+public class PackHandlerProvider extends AbstractHttpHandlerProvider {
+    @Override
+    public HttpHandler[] registerHandlers() {
+        return new HttpHandler[]{
+                new PackHttpHandler()
+        };
+    }
+}

src/main/java/com/fr/plugin/pack/PackURLAliasProvider.java

@@ -0,0 +1,20 @@
+package com.fr.plugin.pack;
+
+import com.fr.decision.fun.impl.AbstractURLAliasProvider;
+import com.fr.decision.webservice.url.alias.URLAlias;
+import com.fr.decision.webservice.url.alias.URLAliasFactory;
+import com.fr.plugin.pack.http.PackHttpHandler;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-19
+ */
+public class PackURLAliasProvider extends AbstractURLAliasProvider {
+    @Override
+    public URLAlias[] registerAlias() {
+        return new URLAlias[]{
+                URLAliasFactory.createPluginAlias(PackHttpHandler.PATH, PackHttpHandler.PATH,false)
+        };
+    }
+}

src/main/java/com/fr/plugin/pack/PackWidgetExecuteProvider.java

@@ -0,0 +1,62 @@
+package com.fr.plugin.pack;
+
+import com.finebi.common.api.vo.table.parameter.Parameters;
+import com.finebi.common.exception.FineEngineException;
+import com.finebi.common.exception.execute.FineWidgetNoDataException;
+import com.finebi.dashboard.api.bean.widget.WidgetBean;
+import com.finebi.dashboard.api.cell.DashboardCellCreator;
+import com.finebi.dashboard.api.fun.AbstractWidgetExecuteProvider;
+import com.finebi.dashboard.api.structure.result.preview.BIPreViewResult;
+import com.finebi.dashboard.impl.widget.EngineResultToPreViewResultVisitor;
+import com.finebi.dashboard.impl.widget.FineWidget;
+import com.finebi.dashboard.impl.widget.visitor.DefaultBeanToWidget;
+import com.finebi.dashboard.impl.widget.visitor.TableAbsentChecker;
+import com.finebi.dashboard.impl.widget.visitor.UrlParameterProcessor;
+import com.finebi.dashboard.impl.widget.visitor.WidgetBeanToFineWidgetVisitor;
+import com.finebi.foundation.api.structure.result.BIResult;
+import com.fr.decision.authority.data.User;
+import com.fr.plugin.pack.exception.WidgetExceedException;
+import com.fr.plugin.pack.traffic.PackTraffic;
+import com.fr.plugin.pack.traffic.PackTrafficFactory;
+
+import java.util.Optional;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-19
+ */
+public class PackWidgetExecuteProvider extends AbstractWidgetExecuteProvider {
+    private PackTraffic widgetTraffic = PackTrafficFactory.getWidgetTraffic();
+
+    @Override
+    public BIPreViewResult getPreViewResult(WidgetBean widgetBean, User user, Parameters parameters) throws FineEngineException {
+        String widgetId = null;
+        try {
+            widgetId = widgetBean.getWid();
+            if (!widgetTraffic.offer(widgetId)) {
+                throw new WidgetExceedException();
+            }
+            return getResult(widgetBean, user, parameters);
+        } finally {
+            widgetTraffic.release(widgetId);
+        }
+    }
+
+    private BIPreViewResult getResult(WidgetBean widgetBean, User user, Parameters parameters) throws FineEngineException {
+        // bean转widget
+        TableAbsentChecker.check(widgetBean);
+        FineWidget widget = new DefaultBeanToWidget<>(new WidgetBeanToFineWidgetVisitor(user))
+                .addProcess(new UrlParameterProcessor(parameters))
+                .translate(widgetBean);
+        // 从引擎拿计算数据
+        BIResult engineResult = getEngineExecutorResult(widget);
+        EngineResultToPreViewResultVisitor resultTranVisitor = new EngineResultToPreViewResultVisitor(engineResult, parameters);
+        return widget.accept(resultTranVisitor);
+    }
+
+    private BIResult getEngineExecutorResult(FineWidget fineWidget) throws FineEngineException {
+        Optional<BIResult> execute = DashboardCellCreator.widgetExecutor().execute(fineWidget);
+        return execute.orElseThrow(() -> new FineWidgetNoDataException("widget result is null ！widget name is : " + fineWidget.getWidgetName()));
+    }
+}

src/main/java/com/fr/plugin/pack/bean/WidgetsBean.java

@@ -0,0 +1,23 @@
+package com.fr.plugin.pack.bean;
+
+import com.finebi.dashboard.api.bean.widget.WidgetBean;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-19
+ */
+public class WidgetsBean {
+    private Map<String, WidgetBean> widgets = new HashMap<>();
+
+    public Map<String, WidgetBean> getWidgets() {
+        return widgets;
+    }
+
+    public void setWidgets(Map<String, WidgetBean> widgets) {
+        this.widgets = widgets;
+    }
+}

src/main/java/com/fr/plugin/pack/conf/PackPluginConfig.java

@@ -0,0 +1,74 @@
+package com.fr.plugin.pack.conf;
+
+import com.fr.config.ConfigContext;
+import com.fr.config.DefaultConfiguration;
+import com.fr.config.Identifier;
+import com.fr.config.Status;
+import com.fr.config.Visualization;
+import com.fr.config.holder.Conf;
+import com.fr.config.holder.factory.Holders;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-20
+ */
+@Visualization(category = "打包插件参数")
+public class PackPluginConfig extends DefaultConfiguration {
+    private static volatile PackPluginConfig config = null;
+
+    // sql查询超时时间(JDBC属性)单位秒
+    @Identifier(value = "sqlQueryTimeout", status = Status.SHOW, name = "SQL查询超时时间", description = "单位秒(JDBC设置queryTimeout)")
+    private Conf<Integer> sqlQueryTimeout = Holders.simple(60);
+
+    // 组件查询超时时间单位毫秒
+    @Identifier(value = "widgetQueryTimeout", status = Status.SHOW, name = "组件查询超时时间", description = "单位毫秒")
+    private Conf<Integer> widgetQueryTimeout = Holders.simple(3 * 60 * 1000);
+
+    // 最大模板并发访问度
+    @Identifier(value = "reportQueryMax", status = Status.SHOW, name = "最大模板并发访问度", description = "模板并发个数")
+    private Conf<Integer> reportQueryMax = Holders.simple(3);
+
+    // 最大组件并发组件并发访问度
+    @Identifier(value = "widgetQueryMax", status = Status.SHOW, name = "最大组件并发组件并发访问度", description = "组件并发个数")
+    private Conf<Integer> widgetQueryMax = Holders.simple(6);
+
+    public static PackPluginConfig getInstance() {
+        if (config == null) {
+            config = ConfigContext.getConfigInstance(PackPluginConfig.class);
+        }
+        return config;
+    }
+
+    public int getSqlQueryTimeout() {
+        return sqlQueryTimeout.get();
+    }
+
+    public void setSqlQueryTimeout(int sqlQueryTimeout) {
+        this.sqlQueryTimeout.set(sqlQueryTimeout);
+    }
+
+    public int getWidgetQueryTimeout() {
+        return widgetQueryTimeout.get();
+    }
+
+    public void setWidgetQueryTimeout(int widgetQueryTimeout) {
+        this.widgetQueryTimeout.set(widgetQueryTimeout);
+    }
+
+    public int getReportQueryMax() {
+        return reportQueryMax.get();
+    }
+
+    public void setReportQueryMax(int reportQueryMax) {
+        this.reportQueryMax.set(reportQueryMax);
+    }
+
+    public int getWidgetQueryMax() {
+        return widgetQueryMax.get();
+    }
+
+    public void setWidgetQueryMax(int widgetQueryMax) {
+        this.widgetQueryMax.set(widgetQueryMax);
+    }
+}

src/main/java/com/fr/plugin/pack/exception/ReportExceedException.java

@@ -0,0 +1,16 @@
+package com.fr.plugin.pack.exception;
+
+import com.finebi.common.exception.FineEngineException;
+import com.fr.plugin.pack.PackConstants;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-19
+ */
+public class ReportExceedException extends FineEngineException {
+    @Override
+    public String errorCode() {
+        return PackConstants.REPORT_EXCEED_ERROR;
+    }
+}

src/main/java/com/fr/plugin/pack/exception/WidgetExceedException.java

@@ -0,0 +1,16 @@
+package com.fr.plugin.pack.exception;
+
+import com.finebi.common.exception.FineEngineException;
+import com.fr.plugin.pack.PackConstants;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-19
+ */
+public class WidgetExceedException extends FineEngineException {
+    @Override
+    public String errorCode() {
+        return PackConstants.WIDGET_EXCEED_ERROR;
+    }
+}

src/main/java/com/fr/plugin/pack/http/PackHttpHandler.java

@@ -0,0 +1,150 @@
+package com.fr.plugin.pack.http;
+
+import com.finebi.activate.foundation.StableManager;
+import com.finebi.base.concurrent.DefaultThreadFactory;
+import com.finebi.common.api.vo.table.parameter.Parameters;
+import com.finebi.dashboard.api.bean.widget.WidgetBean;
+import com.finebi.dashboard.api.service.FineDashBoardService;
+import com.finebi.dashboard.api.service.FineWidgetExecutorService;
+import com.finebi.dashboard.api.structure.result.preview.BIPreViewResult;
+import com.finebi.dashboard.impl.helper.FineDashBoardUtils;
+import com.finebi.foundation.api.reponse.FineRespond;
+import com.finebi.foundation.api.service.FineService;
+import com.finebi.selector.api.DirectSourceEngineFactory;
+import com.finebi.utils.CollectionUtils;
+import com.finebi.utils.StringUtils;
+import com.finebi.web.action.v5.BaseAction;
+import com.fr.decision.authority.data.User;
+import com.fr.decision.fun.impl.BaseHttpHandler;
+import com.fr.general.IOUtils;
+import com.fr.invoke.Reflect;
+import com.fr.log.FineLoggerFactory;
+import com.fr.plugin.pack.bean.WidgetsBean;
+import com.fr.plugin.pack.conf.PackPluginConfig;
+import com.fr.plugin.pack.exception.ReportExceedException;
+import com.fr.plugin.pack.exception.WidgetExceedException;
+import com.fr.plugin.pack.source.PackDBSourceEngine;
+import com.fr.plugin.pack.traffic.PackTraffic;
+import com.fr.plugin.pack.traffic.PackTrafficFactory;
+import com.fr.third.fasterxml.jackson.databind.ObjectMapper;
+import com.fr.third.springframework.web.bind.annotation.RequestMethod;
+import com.fr.web.utils.WebUtils;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-19
+ */
+public class PackHttpHandler extends BaseHttpHandler {
+    private PackTraffic reportTraffic = PackTrafficFactory.getReportTraffic();
+    public static String PATH = "/widgets/data";
+    private ExecutorService executorService = Executors.newCachedThreadPool(new DefaultThreadFactory("pack-widget-execute"));
+
+    {
+        DirectSourceEngineFactory.INSTANCE.register(new PackDBSourceEngine());
+    }
+
+    @Override
+    public RequestMethod getMethod() {
+        return RequestMethod.POST;
+    }
+
+    @Override
+    public String getPath() {
+        return PATH;
+    }
+
+    @Override
+    public boolean isPublic() {
+        return false;
+    }
+
+    @Override
+    public void handle(HttpServletRequest request, HttpServletResponse response) throws Exception {
+        ObjectMapper mapper = new ObjectMapper();
+        String reportId = null;
+        try {
+            String body = IOUtils.inputStream2String(request.getInputStream());
+            WidgetsBean widgetsBean = mapper.readValue(body, WidgetsBean.class);
+
+            Map<String, Object> result = new HashMap<>();
+            Set<String> reportIds = widgetsBean.getWidgets().values().stream().map(WidgetBean::getReportId).collect(Collectors.toSet());
+            if (CollectionUtils.isEmpty(reportIds)) {
+                throw new UnsupportedOperationException();
+            } else {
+                reportId = reportIds.iterator().next();
+                if (StringUtils.isEmpty(reportId)) {
+                    throw new UnsupportedOperationException("reportId must be not emtpy");
+                }
+            }
+            if (!reportTraffic.offer(reportId)) {
+                throw new ReportExceedException();
+            }
+            try {
+                List<Future<?>> futures = new ArrayList<>();
+                for (Map.Entry<String, WidgetBean> entry : widgetsBean.getWidgets().entrySet()) {
+                    WidgetBean widgetBean = entry.getValue();
+                    User user = FineDashBoardUtils.getUserFromReq(request, widgetBean.getReportId());
+                    boolean mobile = FineDashBoardUtils.isMobile(request);
+                    Parameters parameterFromRequest = FineDashBoardUtils.getParameterFromRequest(request);
+                    String widgetId = entry.getKey();
+                    Future<?> future = executorService.submit(() -> {
+                        try {
+                            BIPreViewResult widgetData = Reflect.on(FineDashBoardUtils.class)
+                                    .call("getWidgetData", widgetBean, user, mobile, parameterFromRequest,
+                                            getBean(FineDashBoardService.class), getBean(FineWidgetExecutorService.class))
+                                    .get();
+                            result.put(widgetId, FineRespond.success(widgetData));
+                        } catch (Exception e) {
+                            FineLoggerFactory.getLogger().debug(e.getMessage(), e);
+                            result.put(widgetId, handleException(e));
+                        }
+                    });
+                    futures.add(future);
+                }
+                for (Future<?> future : futures) {
+                    try {
+                        // 设置超时时间，不能一直在等着
+                        future.get(PackPluginConfig.getInstance().getWidgetQueryTimeout(), TimeUnit.MILLISECONDS);
+                    } catch (TimeoutException e) {
+                        future.cancel(true);
+                    }
+                }
+                WebUtils.printAsString(response, mapper.writeValueAsString(FineRespond.success(result)));
+            } finally {
+                reportTraffic.release(reportId);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            WebUtils.printAsString(response, mapper.writeValueAsString(FineRespond.fail("500", mapper.writeValueAsString(handleException(e)))));
+        }
+    }
+
+    private static FineRespond handleException(Exception e) {
+        if (e instanceof ReportExceedException) {
+            return FineRespond.fail(((ReportExceedException) e).errorCode(), e.getMessage());
+        }
+        if (e instanceof WidgetExceedException) {
+            return FineRespond.fail(((WidgetExceedException) e).errorCode(), e.getMessage());
+        }
+        return Reflect.on(BaseAction.class).call("getExceptionRespond", e).get();
+    }
+
+    private static <T extends FineService> T getBean(Class<T> annotatedClass) {
+        return StableManager.getContext().getServiceBean(annotatedClass);
+    }
+}

src/main/java/com/fr/plugin/pack/source/PackDBSourceEngine.java

@@ -0,0 +1,25 @@
+package com.fr.plugin.pack.source;
+
+import com.finebi.api.criterion.Table;
+import com.finebi.base.datasource.ConnectionCreator;
+import com.fr.data.core.db.dialect.Dialect;
+import com.fr.engine.sql.database.DataBaseSourceEngine;
+import com.fr.plugin.pack.conf.PackPluginConfig;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-20
+ */
+public class PackDBSourceEngine extends DataBaseSourceEngine {
+    @Override
+    protected ResultSet executeQuery(Connection conn, Statement statement, String sql, Table table, ConnectionCreator connectionCreator, Dialect dialect) throws SQLException {
+        statement.setQueryTimeout(PackPluginConfig.getInstance().getSqlQueryTimeout());
+        return dialect.executeQuery(statement, sql, conn);
+    }
+}

src/main/java/com/fr/plugin/pack/traffic/PackTraffic.java

@@ -0,0 +1,12 @@
+package com.fr.plugin.pack.traffic;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-19
+ */
+public interface PackTraffic {
+    boolean offer(String id);
+
+    void release(String id);
+}

src/main/java/com/fr/plugin/pack/traffic/PackTrafficFactory.java

@@ -0,0 +1,22 @@
+package com.fr.plugin.pack.traffic;
+
+import com.fr.plugin.pack.PackConstants;
+import com.fr.plugin.pack.conf.PackPluginConfig;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-19
+ */
+public class PackTrafficFactory {
+    private static PackTraffic widgetTraffic = new PackTrafficImpl(PackConstants.WIDGET_TRAFFIC_NAME,() -> PackPluginConfig.getInstance().getWidgetQueryMax());
+    private static PackTraffic reportTraffic = new PackTrafficImpl(PackConstants.REPORT_TRAFFIC_NAME,() -> PackPluginConfig.getInstance().getReportQueryMax());
+
+    public static PackTraffic getWidgetTraffic() {
+        return widgetTraffic;
+    }
+
+    public static PackTraffic getReportTraffic() {
+        return reportTraffic;
+    }
+}

src/main/java/com/fr/plugin/pack/traffic/PackTrafficImpl.java

@@ -0,0 +1,48 @@
+package com.fr.plugin.pack.traffic;
+
+import com.fr.log.FineLoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Supplier;
+
+/**
+ * @author Jonas
+ * @version 5.1.3
+ * Created by Jonas on 2020-10-19
+ */
+public class PackTrafficImpl implements PackTraffic {
+    private Map<String, Integer> mapCount = new ConcurrentHashMap<>();
+    private Supplier<Integer> maxCount;
+    private String name;
+
+    public PackTrafficImpl(String name, Supplier<Integer> maxCount) {
+        this.name = name;
+        this.maxCount = maxCount;
+    }
+
+    @Override
+    public synchronized boolean offer(String id) {
+        Integer count = mapCount.get(id);
+        if (count == null) {
+            count = 0;
+        }
+        int addCount = count + 1;
+        if (addCount > maxCount.get()) {
+            FineLoggerFactory.getLogger().info("PackTraffic({}) false {} addCount {}", name, id, addCount);
+            return false;
+        }
+        mapCount.put(id, addCount);
+        FineLoggerFactory.getLogger().info("PackTraffic({}) true {} addCount {}", name, id, addCount);
+        return true;
+    }
+
+    @Override
+    public synchronized void release(String id) {
+        Integer count = mapCount.get(id);
+        if (count <= 1) {
+            mapCount.remove(id);
+        }
+        mapCount.put(id, count - 1);
+    }
+}