diff --git a/lib/finekit-10.0.jar b/lib/finekit-10.0.jar deleted file mode 100644 index 546f2f5..0000000 Binary files a/lib/finekit-10.0.jar and /dev/null differ diff --git a/plugin.xml b/plugin.xml index 3dd83a2..34fd8ac 100644 --- a/plugin.xml +++ b/plugin.xml @@ -3,12 +3,13 @@ com.fr.plugin.db.es.v10 yes - 1.5 + 1.5.5 10.0 2019-10-25 fanruan.richie [2020-03-18]默认使用$..*查询语句获取所有值。
[2020-03-11]新增查询请求类型。
[2020-03-07]远程设计序列化问题。
diff --git a/src/main/java/com/fr/plugin/db/es/fun/ElasticsearchDataModel.java b/src/main/java/com/fr/plugin/db/es/fun/ElasticsearchDataModel.java index 183d00c..2a12f5c 100644 --- a/src/main/java/com/fr/plugin/db/es/fun/ElasticsearchDataModel.java +++ b/src/main/java/com/fr/plugin/db/es/fun/ElasticsearchDataModel.java @@ -3,6 +3,9 @@ package com.fr.plugin.db.es.fun; import com.fanruan.api.data.open.BaseDataModel; import com.fanruan.api.err.TableDataException; import com.fanruan.api.util.StringKit; +import com.fr.json.JSON; +import com.fr.json.JSONFactory; +import com.fr.json.JSONObject; import com.fr.plugin.db.es.fun.assist.SimpleDataModel; import com.fr.plugin.db.es.fun.category.ResultStandardizeSelector; import com.fr.plugin.db.es.fun.type.ConverterType; @@ -77,7 +80,7 @@ public class ElasticsearchDataModel extends BaseDataModel { String text; if (StringKit.isNotBlank(endPoint)) { Request request = new Request(queryType.getType(), endPoint); - request.setJsonEntity(query); + request.setJsonEntity(JSONFactory.createJSON(JSON.OBJECT, query).toString()); Response response = client.performRequest(request); text = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); } else { diff --git a/src/main/java/com/fr/plugin/db/es/fun/ElasticsearchTableData.java b/src/main/java/com/fr/plugin/db/es/fun/ElasticsearchTableData.java index 80532f8..331068c 100644 --- a/src/main/java/com/fr/plugin/db/es/fun/ElasticsearchTableData.java +++ b/src/main/java/com/fr/plugin/db/es/fun/ElasticsearchTableData.java @@ -1,8 +1,10 @@ package com.fr.plugin.db.es.fun; +import com.fanruan.api.cal.ParameterKit; import com.fanruan.api.conf.HolderKit; import com.fanruan.api.data.ConnectionKit; import com.fanruan.api.data.open.BaseTableData; +import com.fanruan.api.log.LogKit; import com.fanruan.api.util.AssistKit; import com.fanruan.api.util.StringKit; import com.fanruan.api.xml.XmlKit; @@ -12,17 +14,28 @@ import com.fr.config.holder.Conf; import com.fr.data.impl.Connection; import com.fr.general.data.DataModel; import com.fr.intelli.record.Focus; +import com.fr.log.FineLoggerFactory; import com.fr.plugin.db.es.fun.help.RenderUtils; +import com.fr.plugin.db.es.fun.query.QueryBuilder; import com.fr.plugin.db.es.fun.type.ConverterType; +import com.fr.plugin.db.es.fun.type.ParaType; import com.fr.plugin.db.es.fun.type.QueryType; import com.fr.record.analyzer.EnableMetrics; import com.fr.script.Calculator; import com.fr.stable.NameReference; import com.fr.stable.ParameterProvider; +import com.fr.stable.StringUtils; import com.fr.stable.xml.XMLPrintWriter; import com.fr.stable.xml.XMLableReader; +import com.fr.third.fasterxml.jackson.annotation.JsonInclude; +import com.fr.third.fasterxml.jackson.databind.JsonNode; +import com.fr.third.fasterxml.jackson.databind.ObjectMapper; +import com.fr.third.fasterxml.jackson.databind.SerializationFeature; +import java.util.Arrays; import java.util.Collection; +import java.util.Iterator; +import java.util.concurrent.atomic.AtomicInteger; /** * @author richie @@ -38,6 +51,8 @@ public class ElasticsearchTableData extends BaseTableData { private Conf database = HolderKit.obj(null, Connection.class); @Identifier("endPoint") private Conf endPoint = HolderKit.simple(StringKit.EMPTY); + @Identifier("paraType") + private Conf paraType = HolderKit.simple(ParaType.PLAIN.getType()); @Identifier("query") private Conf query = HolderKit.simple(StringKit.EMPTY); @Identifier("queryType") @@ -69,6 +84,14 @@ public class ElasticsearchTableData extends BaseTableData { this.endPoint.set(endPoint); } + public ParaType getParaType() { + return ParaType.parse(paraType.get()); + } + + public void setParaType(ParaType paraType) { + this.paraType.set(paraType.getType()); + } + public String getQuery() { return query.get(); } @@ -118,17 +141,21 @@ public class ElasticsearchTableData extends BaseTableData { @Focus(id = "com.fr.plugin.db.es.v10", text = "Plugin_Elasticsearch_Table_Data") public DataModel createDataModel(Calculator calculator, int rowCount) { ElasticsearchConnection connection = getConnection(); - Collection parameterCollection = this.parameters.get(); - ParameterProvider[] parameterProviders = (ParameterProvider[]) parameterCollection.toArray(new ParameterProvider[0]); + Collection parameterCollection = this.parameters.get(); + if (parameterCollection.isEmpty()) { + this.parameters.set(Arrays.asList(ParameterKit.analyze4Parameters(query.get(), false))); + } + ParameterProvider[] parameterProviders = processParameters(calculator); ParameterProvider[] ps = Calculator.processParameters(calculator, parameterProviders); + String realQueryText = new QueryBuilder().build(ps, query.get(), getParaType()); if (connection != null) { return new ElasticsearchDataModel( calculator, connection, - RenderUtils.calculateQuery(endPoint.get(), ps), - RenderUtils.calculateQuery(query.get(), ps), + RenderUtils.renderPlainText(endPoint.get(), ps), + realQueryText, getQueryType(), - RenderUtils.calculateQuery(script.get(), ps), + RenderUtils.renderPlainText(script.get(), ps), getConverterType(), getConfigAttribute(), rowCount @@ -137,6 +164,49 @@ public class ElasticsearchTableData extends BaseTableData { return null; } + private String removeEmptyQuery(String realQueryText) { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + mapper.configure(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS, false); + mapper.configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + mapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + + String jsonText = realQueryText.replaceAll(":\\s*}", ":null}"); + + try { + JsonNode obj = mapper.readTree(jsonText); + AtomicInteger status = new AtomicInteger(0); + int i; + do { + i = stripEmpty(obj, status); + status.decrementAndGet(); + } while (i != 0); + String txt = mapper.writeValueAsString(obj); + LogKit.info("[Elasticsearch]Query is:{}", txt); + return txt; + } catch (Exception e) { + LogKit.error(e, "[Elasticsearch]{}", e.getMessage()); + } + return realQueryText; + } + + private int stripEmpty(JsonNode node, AtomicInteger counter) { + Iterator it = node.iterator(); + while (it.hasNext()) { + JsonNode child = it.next(); + if (child.isNull() + || child.isEmpty(null) + || (child.isTextual() && StringUtils.isEmpty(child.asText()))) { + it.remove(); + counter.incrementAndGet(); + } else { + stripEmpty(child, counter); + } + } + return counter.get(); + } + private ElasticsearchConnection getConnection() { if (database.get() instanceof NameReference) { String name = ((NameReference) database.get()).getName(); @@ -156,6 +226,10 @@ public class ElasticsearchTableData extends BaseTableData { Connection con = XmlKit.readXMLConnection(reader); this.setDatabase(con); } + } else if ("ParaType".equals(tmpName)) { + if ((tmpVal = reader.getElementValue()) != null) { + this.setParaType(ParaType.parse(tmpVal)); + } } else if ("Query".equals(tmpName)) { if ((tmpVal = reader.getElementValue()) != null) { this.setQuery(tmpVal); @@ -186,6 +260,7 @@ public class ElasticsearchTableData extends BaseTableData { if (configAttribute.get() != null) { XmlKit.writeXMLable(writer, configAttribute.get(), ConfigAttribute.XML_TAG); } + writer.startTAG("ParaType").textNode(getParaType().getType()).end(); writer.startTAG("Query").textNode(getQuery()).end(); writer.startTAG("QueryType").textNode(getQueryType().getType()).end(); writer.startTAG("Script").textNode(getScript()).end(); @@ -200,6 +275,7 @@ public class ElasticsearchTableData extends BaseTableData { ElasticsearchTableData cloned = (ElasticsearchTableData) super.clone(); cloned.database = (Conf) database.clone(); cloned.endPoint = (Conf) endPoint.clone(); + cloned.paraType = (Conf) paraType.clone(); cloned.converterType = (Conf) converterType.clone(); cloned.query = (Conf) query.clone(); cloned.queryType = (Conf) queryType.clone(); @@ -212,13 +288,17 @@ public class ElasticsearchTableData extends BaseTableData { return obj instanceof ElasticsearchTableData && AssistKit.equals(database, ((ElasticsearchTableData) obj).database) && AssistKit.equals(endPoint, ((ElasticsearchTableData) obj).endPoint) + && AssistKit.equals(converterType, ((ElasticsearchTableData) obj).converterType) + + && AssistKit.equals(paraType, ((ElasticsearchTableData) obj).paraType) + && AssistKit.equals(query, ((ElasticsearchTableData) obj).query) && AssistKit.equals(script, ((ElasticsearchTableData) obj).script); } @Override public int hashCode() { - return AssistKit.hashCode(database.get(), endPoint.get(), converterType.get(), query.get(), script.get()); + return AssistKit.hashCode(database.get(), endPoint.get(), converterType.get(), paraType.get(), query.get(), script.get()); } } diff --git a/src/main/java/com/fr/plugin/db/es/fun/help/RenderUtils.java b/src/main/java/com/fr/plugin/db/es/fun/help/RenderUtils.java index adc5234..a879d13 100644 --- a/src/main/java/com/fr/plugin/db/es/fun/help/RenderUtils.java +++ b/src/main/java/com/fr/plugin/db/es/fun/help/RenderUtils.java @@ -1,8 +1,12 @@ package com.fr.plugin.db.es.fun.help; +import com.fanruan.api.script.ScriptKit; import com.fanruan.api.util.ArrayKit; import com.fanruan.api.util.RenderKit; import com.fr.stable.ParameterProvider; +import com.fr.third.fasterxml.jackson.annotation.JsonInclude; +import com.fr.third.fasterxml.jackson.databind.ObjectMapper; +import com.fr.third.fasterxml.jackson.databind.SerializationFeature; import java.util.HashMap; import java.util.Map; @@ -14,7 +18,7 @@ import java.util.Map; */ public class RenderUtils { - public static String calculateQuery(String query, ParameterProvider[] ps) { + public static String renderPlainText(String query, ParameterProvider[] ps) { if (ArrayKit.isEmpty(ps)) { return query; } @@ -28,4 +32,38 @@ public class RenderUtils { return query; } } + + public static String clearEmptyQueryNode(String query) { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + mapper.configure(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS, false); + mapper.configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + mapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + + String jsonText = query.replaceAll(":}", ":null}"); + try { + Object obj = mapper.readValue(jsonText, Object.class); + String r = mapper.writeValueAsString(obj); + Object rt = ScriptKit.newScriptEngine().eval(String.format("(function ($) {\n" + + " function clearEmpties(o) {\n" + + " for (var k in o) {\n" + + " if (!o[k] || typeof o[k] !== \"object\") {\n" + + " continue\n" + + " }\n" + + " clearEmpties(o[k]);\n" + + " if (Object.keys(o[k]).length === 0) {\n" + + " delete o[k];\n" + + " }\n" + + " }\n" + + " }\n" + + " var obj = JSON.parse($);\n" + + " clearEmpties(obj);\n" + + " return obj;\n" + + "})('%s')", r)); + return mapper.writeValueAsString(rt); + } catch (Exception e) { + return query; + } + } } diff --git a/src/main/java/com/fr/plugin/db/es/fun/query/QueryBuilder.java b/src/main/java/com/fr/plugin/db/es/fun/query/QueryBuilder.java new file mode 100644 index 0000000..ae7ccfd --- /dev/null +++ b/src/main/java/com/fr/plugin/db/es/fun/query/QueryBuilder.java @@ -0,0 +1,93 @@ +package com.fr.plugin.db.es.fun.query; + +import com.fanruan.api.cal.ParameterKit; +import com.fanruan.api.log.LogKit; +import com.fanruan.api.script.ScriptKit; +import com.fanruan.api.util.GeneralKit; +import com.fanruan.api.util.RenderKit; +import com.fr.general.FArray; +import com.fr.plugin.db.es.fun.help.RenderUtils; +import com.fr.plugin.db.es.fun.type.ParaType; +import com.fr.stable.ArrayUtils; +import com.fr.stable.ParameterProvider; + +import javax.script.ScriptException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author richie + * @version 10.0 + * Created by richie on 2020/9/11 + */ +public class QueryBuilder { + + public String build(ParameterProvider[] ps, String query, ParaType paraType) { + if (ParaType.PLAIN == paraType) { + return RenderUtils.renderPlainText(query, ps); + } + int len = ArrayUtils.getLength(ps); + StringBuilder sb = new StringBuilder(); + sb.append("JSON.stringify((function("); + String[] paras = new String[len]; + Object[] values = new Object[len]; + for (int i = 0; i < len;i ++) { + paras[i] = ps[i].getName(); + Object value = ps[i].getValue(); + if (value instanceof String) { + values[i] = "\"" + value + "\""; + } else if (value instanceof Iterable) { + List objects = new ArrayList<>(); + ((Iterable)value).forEach(el-> { + if (el instanceof String) { + objects.add("\"" + el + "\""); + } else { + objects.add(el); + } + }); + values[i] = "[" + GeneralKit.join(objects.toArray(new Object[0]), ",") + "]"; + } else { + values[i] = value; + } + } + sb.append(GeneralKit.join(paras, ",")); + sb.append("){"); + sb.append(renderQuery(ps, query)); + sb.append("}("); + sb.append(GeneralKit.join(values, ",")); + sb.append(")))"); + try { + return GeneralKit.objectToString(ScriptKit.newScriptEngine().eval(sb.toString())); + } catch (ScriptException e) { + LogKit.error(e.getMessage(), e); + } + return RenderUtils.renderPlainText(query, ps); + } + + private String renderQuery(ParameterProvider[] ps, String query) { + int len = ArrayUtils.getLength(ps); + Map map = new HashMap<>(); + for (int i = 0; i < len; i ++) { + map.put(ps[i].getName(), ps[i].getName()); + } + try { + return RenderKit.renderParameter4Tpl(query, map); + } catch (Exception e) { + return query; + } + } + + public static void main(String... args) { + ParameterProvider[] ps = new ParameterProvider[] { + ParameterKit.newParameter("aa", 1), + ParameterKit.newParameter("bb", "xxx"), + ParameterKit.newParameter("cc", null), + ParameterKit.newParameter("dd", new FArray(new String[]{"qqq","qq","zzz"})), + }; + QueryBuilder builder = new QueryBuilder(); + String r = builder.build(ps, "return {\"aa\":aa};", ParaType.ECMASCRIPT); + System.out.println(r); + } +} diff --git a/src/main/java/com/fr/plugin/db/es/fun/type/ParaType.java b/src/main/java/com/fr/plugin/db/es/fun/type/ParaType.java new file mode 100644 index 0000000..236b5ed --- /dev/null +++ b/src/main/java/com/fr/plugin/db/es/fun/type/ParaType.java @@ -0,0 +1,30 @@ +package com.fr.plugin.db.es.fun.type; + +/** + * @author richie + * @version 10.0 + * Created by richie on 2020/9/11 + */ +public enum ParaType { + + PLAIN("PLAIN"), ECMASCRIPT("SCRIPT"); + + private String type; + + ParaType(String type) { + this.type = type; + } + + public String getType() { + return type; + } + + public static ParaType parse(String text) { + for (ParaType qt : values()) { + if (qt.type.equals(text)) { + return qt; + } + } + return PLAIN; + } +} diff --git a/src/main/java/com/fr/plugin/db/es/ui/ElasticsearchQueryPane.java b/src/main/java/com/fr/plugin/db/es/ui/ElasticsearchQueryPane.java index 6725b39..8fb804f 100644 --- a/src/main/java/com/fr/plugin/db/es/ui/ElasticsearchQueryPane.java +++ b/src/main/java/com/fr/plugin/db/es/ui/ElasticsearchQueryPane.java @@ -13,6 +13,7 @@ import com.fanruan.api.design.ui.layout.TableLayoutKit; import com.fr.design.dialog.BasicPane; import com.fr.plugin.db.es.fun.ConfigAttribute; import com.fr.plugin.db.es.fun.type.ConverterType; +import com.fr.plugin.db.es.fun.type.ParaType; import com.fr.plugin.db.es.fun.type.QueryType; import javax.swing.*; @@ -30,6 +31,8 @@ public class ElasticsearchQueryPane extends BasicPane { private UITextField endPointTextField; private UIDictionaryComboBox converterTypeComboBox; private UIDictionaryComboBox queryTypeComboBox; + private UIDictionaryComboBox paraTypeComboBox; + private UISyntaxTextArea queryTextPane; private UISyntaxTextArea scriptTextPane; private UIDictionaryComboBox sortedComboBox; @@ -46,7 +49,7 @@ public class ElasticsearchQueryPane extends BasicPane { double p = TableLayoutKit.PREFERRED; double f = TableLayoutKit.FILL; - double[] rowSize = {p, p, p, p, p, p, p}; + double[] rowSize = {p, p, p, p, p, p, p, p}; double[] columnSize = {p, f}; endPointTextField = new UITextField(); @@ -55,6 +58,14 @@ public class ElasticsearchQueryPane extends BasicPane { new QueryType[]{QueryType.GET, QueryType.POST}, new String[]{QueryType.GET.getType(), QueryType.POST.getType()}); + paraTypeComboBox = new UIDictionaryComboBox<>( + new ParaType[]{ParaType.PLAIN, ParaType.ECMASCRIPT}, + new String[] { + DesignKit.i18nText("Plugin-Elasticsearch_Para_Type_Plain"), + DesignKit.i18nText("Plugin-Elasticsearch_Para_Type_ECMAScript") + } + ); + queryTextPane = new UISyntaxTextArea(); converterTypeComboBox = new UIDictionaryComboBox<>(new ConverterType[]{ @@ -75,14 +86,15 @@ public class ElasticsearchQueryPane extends BasicPane { sortedComboBox = new UIDictionaryComboBox<>( new Boolean[]{false, true}, - new String[]{DesignKit.i18nText("Plugin-Elasticsearch_Sort_Default"), com.fr.design.i18n.Toolkit.i18nText("Plugin-JSON_Sort_Open")}); + new String[]{DesignKit.i18nText("Plugin-Elasticsearch_Sort_Default"), com.fr.design.i18n.Toolkit.i18nText("Plugin-Elasticsearch_Sort_Open")}); prepareComboBox = new UIDictionaryComboBox<>( new Boolean[]{false, true}, - new String[]{DesignKit.i18nText("Plugin-Elasticsearch_Prepare_Keys_Default"), com.fr.design.i18n.Toolkit.i18nText("Plugin-JSON_Prepare_Keys_Open")} + new String[]{DesignKit.i18nText("Plugin-Elasticsearch_Prepare_Keys_Default"), com.fr.design.i18n.Toolkit.i18nText("Plugin-Elasticsearch_Prepare_Keys_Open")} ); Component[][] coms = new Component[][]{ {new UILabel(DesignKit.i18nText("Plugin-Elasticsearch_Endpoint") + ":"), endPointTextField}, {new UILabel(DesignKit.i18nText("Plugin-Elasticsearch_Query_Type") + ":"), queryTypeComboBox}, + {new UILabel(DesignKit.i18nText("Plugin-Elasticsearch_Para_Type") + ":"), paraTypeComboBox}, {new UILabel(DesignKit.i18nText("Plugin-Elasticsearch_Query") + ":"), createConditionTextPane(queryTextPane)}, {new UILabel(DesignKit.i18nText("Plugin-Elasticsearch_Result_Type")+ ":"), converterTypeComboBox}, {new UILabel(DesignKit.i18nText("Plugin-Elasticsearch_Result_Script")+ ":"), createConditionTextPane(scriptTextPane)}, @@ -123,6 +135,14 @@ public class ElasticsearchQueryPane extends BasicPane { converterTypeComboBox.setSelectedItem(converterType); } + public ParaType getParaType() { + return paraTypeComboBox.getSelectedItem(); + } + + public void setParaType(ParaType paraType) { + paraTypeComboBox.setSelectedItem(paraType); + } + public QueryType getQueryType() { return queryTypeComboBox.getSelectedItem(); } diff --git a/src/main/java/com/fr/plugin/db/es/ui/ElasticsearchTableDataPane.java b/src/main/java/com/fr/plugin/db/es/ui/ElasticsearchTableDataPane.java index 25252d3..7910c61 100644 --- a/src/main/java/com/fr/plugin/db/es/ui/ElasticsearchTableDataPane.java +++ b/src/main/java/com/fr/plugin/db/es/ui/ElasticsearchTableDataPane.java @@ -164,6 +164,7 @@ public class ElasticsearchTableDataPane extends BaseTableDataPane