forked from fanruan/design
Browse Source
* commit '57283ecd9904109f2e31aaf0643d25050ecf132a': 用IOUtils.zip压缩文件 改下命名 使用websocket处理前端发来的埋点消息； 根据产品新规定的埋点标识命名； 文件操作放在FileEntityBuilder里； 代码质量 fix 数据连接埋点及回传 优化取堆栈记录逻辑research/10.0
6 changed files with 459 additions and 259 deletions
@ -0,0 +1,149 @@
|
||||
package com.fr.design.mainframe.messagecollect.entity; |
||||
|
||||
import com.fr.general.CloudCenter; |
||||
import com.fr.general.IOUtils; |
||||
import com.fr.general.http.HttpRequestType; |
||||
import com.fr.general.http.HttpToolbox; |
||||
import com.fr.json.JSONArray; |
||||
import com.fr.json.JSONException; |
||||
import com.fr.json.JSONObject; |
||||
import com.fr.log.FineLoggerFactory; |
||||
import com.fr.stable.CommonUtils; |
||||
import com.fr.stable.EncodeConstants; |
||||
import com.fr.stable.ProductConstants; |
||||
import com.fr.stable.StableUtils; |
||||
import com.fr.stable.StringUtils; |
||||
import com.fr.third.org.apache.http.entity.mime.MultipartEntityBuilder; |
||||
import com.fr.third.org.apache.http.entity.mime.content.FileBody; |
||||
|
||||
import java.io.ByteArrayInputStream; |
||||
import java.io.File; |
||||
import java.io.FileOutputStream; |
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.nio.charset.Charset; |
||||
import java.util.HashMap; |
||||
import java.util.Map; |
||||
|
||||
/** |
||||
* @author alex sung |
||||
* @date 2019/4/8 |
||||
*/ |
||||
public class FileEntityBuilder { |
||||
|
||||
private static final String INTELLI_OPERATION_URL = "intelli.operation.url"; |
||||
private static final String OPERATION_URL = "https://cloud.fanruan.com/config/protect/operation"; |
||||
private static final String ATTR_SIGNATURE = "signature"; |
||||
private static final String ATTR_KEY = "key"; |
||||
/** |
||||
* 文件名 |
||||
*/ |
||||
private String fileName; |
||||
/** |
||||
* 文件的完整路径 |
||||
*/ |
||||
private String pathName; |
||||
/** |
||||
* 文件夹路径 |
||||
*/ |
||||
private String folderName; |
||||
|
||||
public FileEntityBuilder(String fileName, String pathName, String folderName) { |
||||
this.fileName = fileName; |
||||
this.pathName = pathName; |
||||
this.folderName = folderName; |
||||
} |
||||
|
||||
public String getFileName() { |
||||
return fileName; |
||||
} |
||||
|
||||
public void setFileName(String fileName) { |
||||
this.fileName = fileName; |
||||
} |
||||
|
||||
public String getPathName() { |
||||
return pathName; |
||||
} |
||||
|
||||
public void setPathName(String pathName) { |
||||
this.pathName = pathName; |
||||
} |
||||
|
||||
public String getFolderName() { |
||||
return folderName; |
||||
} |
||||
|
||||
public void setFolderName(String folderName) { |
||||
this.folderName = folderName; |
||||
} |
||||
|
||||
public File generateZipFile(String pathName) { |
||||
File zipFile = null; |
||||
try { |
||||
zipFile = new File(pathName + ".zip"); |
||||
java.util.zip.ZipOutputStream zipOut = new java.util.zip.ZipOutputStream(new FileOutputStream(zipFile)); |
||||
IOUtils.zip(zipOut, new File(pathName)); |
||||
zipOut.close(); |
||||
} catch (Exception e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||
} |
||||
return zipFile; |
||||
} |
||||
|
||||
public void generateFile(JSONArray jsonArray, String pathName) { |
||||
try { |
||||
String content = jsonArray.toString(); |
||||
File file = new File(pathName + ".json"); |
||||
StableUtils.makesureFileExist(file); |
||||
FileOutputStream out = new FileOutputStream(file); |
||||
InputStream in = new ByteArrayInputStream(content.getBytes(EncodeConstants.ENCODING_UTF_8)); |
||||
IOUtils.copyBinaryTo(in, out); |
||||
out.close(); |
||||
} catch (Exception e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||
} |
||||
} |
||||
|
||||
public void deleteFileAndZipFile(File zipFile, String pathName) { |
||||
File file = new File(StableUtils.pathJoin(ProductConstants.getEnvHome(), pathName)); |
||||
CommonUtils.deleteFile(file); |
||||
CommonUtils.deleteFile(zipFile); |
||||
} |
||||
|
||||
/** |
||||
* 上传文件到云中心 |
||||
* @param file 待上传文件 |
||||
* @param keyFileName 目标文件 |
||||
* @throws IOException |
||||
*/ |
||||
public static void uploadFile(File file, String keyFileName) throws IOException { |
||||
String url = generateSignedUploadUrl("FocusPoint/"+keyFileName); |
||||
if(StringUtils.isEmpty(url)){ |
||||
FineLoggerFactory.getLogger().error("url is null."); |
||||
}else { |
||||
MultipartEntityBuilder builder = MultipartEntityBuilder.create() |
||||
.addPart("file", new FileBody(file)); |
||||
Map<String, String> headers = new HashMap<String, String>(); |
||||
headers.put("Content-Type", "application/zip"); |
||||
HttpToolbox.upload(url, builder, Charset.forName("utf-8"), headers, HttpRequestType.PUT); |
||||
} |
||||
} |
||||
|
||||
private static String generateSignedUploadUrl(String fileKeyName) throws IOException { |
||||
String url = CloudCenter.getInstance().acquireUrlByKind(INTELLI_OPERATION_URL, OPERATION_URL); |
||||
Map<String, String> parameters = new HashMap<String, String>(); |
||||
parameters.put(ATTR_KEY, fileKeyName); |
||||
parameters.put(ATTR_SIGNATURE, String.valueOf(CommonUtils.signature())); |
||||
String responseText = HttpToolbox.get(url, parameters); |
||||
try { |
||||
JSONObject data = new JSONObject(responseText); |
||||
if ("success".equals(data.optString("status"))) { |
||||
return data.optString("url"); |
||||
} |
||||
} catch (JSONException e) { |
||||
FineLoggerFactory.getLogger().error("Illegal response text."+e, e.getMessage()); |
||||
} |
||||
return null; |
||||
} |
||||
} |
||||
@ -0,0 +1,133 @@
|
||||
package com.fr.design.mainframe.messagecollect.impl; |
||||
|
||||
import com.fr.design.mainframe.messagecollect.entity.FileEntityBuilder; |
||||
import com.fr.design.mainframe.messagecollect.utils.MessageCollectUtils; |
||||
import com.fr.intelli.record.MetricRegistry; |
||||
import com.fr.json.JSONArray; |
||||
import com.fr.log.FineLoggerFactory; |
||||
import com.fr.stable.ProductConstants; |
||||
import com.fr.stable.StableUtils; |
||||
import com.fr.stable.query.QueryFactory; |
||||
import com.fr.stable.query.condition.QueryCondition; |
||||
import com.fr.stable.query.data.DataList; |
||||
import com.fr.stable.query.restriction.RestrictionFactory; |
||||
import com.fr.stable.xml.XMLTools; |
||||
import com.fr.stable.xml.XMLable; |
||||
|
||||
import java.io.File; |
||||
import java.io.FileOutputStream; |
||||
|
||||
/** |
||||
* @author alex sung |
||||
* @date 2019/3/22 |
||||
*/ |
||||
public abstract class AbstractSendDataToCloud implements XMLable { |
||||
private static final String FILE_NAME = "messagecollect.info"; |
||||
private static final String COLUMN_TIME = "time"; |
||||
|
||||
protected String lastTime; |
||||
private static final int PAGE_SIZE = 200; |
||||
private long totalCount = -1; |
||||
private FileEntityBuilder fileEntityBuilder; |
||||
|
||||
public FileEntityBuilder getFileEntityBuilder() { |
||||
return fileEntityBuilder; |
||||
} |
||||
|
||||
public void setFileEntityBuilder(FileEntityBuilder fileEntityBuilder) { |
||||
this.fileEntityBuilder = fileEntityBuilder; |
||||
} |
||||
|
||||
public String getLastTime() { |
||||
return lastTime; |
||||
} |
||||
|
||||
public void setLastTime(String lastTime) { |
||||
this.lastTime = lastTime; |
||||
} |
||||
|
||||
public void saveLastTime() { |
||||
setLastTime(MessageCollectUtils.dateToString()); |
||||
try { |
||||
FileOutputStream out = new FileOutputStream(getLastTimeFile()); |
||||
XMLTools.writeOutputStreamXML(this, out); |
||||
} catch (Exception ex) { |
||||
FineLoggerFactory.getLogger().error(ex.getMessage()); |
||||
} |
||||
} |
||||
|
||||
public static File getLastTimeFile() { |
||||
return new File(StableUtils.pathJoin(ProductConstants.getEnvHome(), FILE_NAME)); |
||||
} |
||||
|
||||
public <T> void queryData(long currentTime, long lastTime, Class<T> tClass) { |
||||
queryAndSendOnePageFunctionContent(currentTime, lastTime, 0, tClass); |
||||
long page = (totalCount / PAGE_SIZE) + 1; |
||||
for (int i = 1; i < page; i++) { |
||||
queryAndSendOnePageFunctionContent(currentTime, lastTime, i, tClass); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public Object clone() throws CloneNotSupportedException { |
||||
return super.clone(); |
||||
} |
||||
|
||||
private <T> void queryAndSendOnePageFunctionContent(long current, long last, int page, Class<T> tClass) { |
||||
QueryCondition condition = QueryFactory.create() |
||||
.skip(page * PAGE_SIZE) |
||||
.count(PAGE_SIZE) |
||||
.addSort(COLUMN_TIME, true) |
||||
.addRestriction(RestrictionFactory.lte(COLUMN_TIME, current)) |
||||
.addRestriction(RestrictionFactory.gte(COLUMN_TIME, last)); |
||||
try { |
||||
DataList<T> points = MetricRegistry.getMetric().find(tClass, condition); |
||||
//第一次查询获取总记录数
|
||||
if (page == 0) { |
||||
totalCount = points.getTotalCount(); |
||||
} |
||||
dealWithData(points); |
||||
|
||||
} catch (Exception e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||
} |
||||
} |
||||
|
||||
public <T> void dealWithData(DataList<T> tDataList) throws Exception { |
||||
generateThisPageFile(tDataList); |
||||
} |
||||
|
||||
private <T> void generateThisPageFile(DataList<T> points) { |
||||
File file = null; |
||||
try { |
||||
JSONArray jsonArray = dealWithSendFunctionContent(points); |
||||
//生成json文件
|
||||
fileEntityBuilder.generateFile(jsonArray, getFileEntityBuilder().getPathName()); |
||||
} catch (Exception e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||
} |
||||
} |
||||
|
||||
public abstract <T> JSONArray dealWithSendFunctionContent(DataList<T> focusPoints); |
||||
|
||||
/** |
||||
* 生成zip并发送zip文件 |
||||
* @param pathName zip文件路径 |
||||
*/ |
||||
protected void sendZipFile(String pathName) { |
||||
|
||||
File file = null; |
||||
try { |
||||
file = fileEntityBuilder.generateZipFile(pathName); |
||||
if (file != null) { |
||||
fileEntityBuilder.uploadFile(file, file.getName()); |
||||
} |
||||
} catch (Exception e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||
return; |
||||
} |
||||
fileEntityBuilder.deleteFileAndZipFile(file, pathName); |
||||
} |
||||
|
||||
|
||||
} |
||||
@ -0,0 +1,105 @@
|
||||
package com.fr.design.mainframe.messagecollect.impl; |
||||
|
||||
import com.fr.config.MarketConfig; |
||||
import com.fr.design.DesignerEnvManager; |
||||
import com.fr.design.mainframe.messagecollect.entity.FileEntityBuilder; |
||||
import com.fr.design.mainframe.messagecollect.utils.MessageCollectUtils; |
||||
import com.fr.intelli.record.FocusPoint; |
||||
import com.fr.json.JSONArray; |
||||
import com.fr.json.JSONObject; |
||||
import com.fr.log.FineLoggerFactory; |
||||
import com.fr.stable.ProductConstants; |
||||
import com.fr.stable.StableUtils; |
||||
import com.fr.stable.query.data.DataList; |
||||
import com.fr.stable.xml.XMLPrintWriter; |
||||
import com.fr.stable.xml.XMLableReader; |
||||
|
||||
import java.util.Date; |
||||
import java.util.UUID; |
||||
|
||||
/** |
||||
* @author alex sung |
||||
* @date 2019/3/22 |
||||
*/ |
||||
public class FocusPointMessageUploader extends AbstractSendDataToCloud { |
||||
|
||||
private static final String TAG = "FocusPointMessageTag"; |
||||
private static final String SEPARATOR = "_"; |
||||
private static final String FOCUS_POINT = "FocusPoint"; |
||||
private static volatile FocusPointMessageUploader instance; |
||||
|
||||
public static FocusPointMessageUploader getInstance() { |
||||
if (instance == null) { |
||||
synchronized (FocusPointMessageUploader.class) { |
||||
if (instance == null) { |
||||
instance = new FocusPointMessageUploader(); |
||||
} |
||||
} |
||||
} |
||||
return instance; |
||||
} |
||||
|
||||
@Override |
||||
public <T> JSONArray dealWithSendFunctionContent(DataList<T> focusPoints) { |
||||
JSONArray ja = new JSONArray(); |
||||
for(T t:focusPoints.getList()){ |
||||
FocusPoint focusPoint = (FocusPoint)t; |
||||
JSONObject jo = new JSONObject(); |
||||
jo.put("id",focusPoint.getId()); |
||||
jo.put("text",focusPoint.getId()); |
||||
jo.put("source",focusPoint.getId()); |
||||
jo.put("time",focusPoint.getId()); |
||||
jo.put("username",focusPoint.getId()); |
||||
jo.put("ip",focusPoint.getId()); |
||||
jo.put("title",focusPoint.getId()); |
||||
jo.put("body",focusPoint.getId()); |
||||
ja.put(jo); |
||||
} |
||||
return ja; |
||||
} |
||||
|
||||
public void sendToCloudCenter() { |
||||
MessageCollectUtils.readXMLFile(instance, getLastTimeFile()); |
||||
long currentTime = new Date().getTime(); |
||||
long lastTIme = MessageCollectUtils.getLastTimeMillis(lastTime); |
||||
try { |
||||
generatePath(); |
||||
queryData(currentTime, lastTIme, FocusPoint.class); |
||||
} catch (Exception e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage()); |
||||
} |
||||
sendZipFile(getFileEntityBuilder().getFolderName()); |
||||
saveLastTime(); |
||||
} |
||||
|
||||
@Override |
||||
public void readXML(XMLableReader reader) { |
||||
if (reader.isAttr()) { |
||||
this.setLastTime(reader.getAttrAsString("focusPointLastTime", null)); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public void writeXML(XMLPrintWriter writer) { |
||||
writer.startTAG(TAG); |
||||
writer.attr("focusPointLastTime", lastTime); |
||||
writer.end(); |
||||
} |
||||
|
||||
private void generatePath() { |
||||
DesignerEnvManager envManager = DesignerEnvManager.getEnvManager(); |
||||
String bbsUserName = MarketConfig.getInstance().getBbsUsername(); |
||||
String uuid = envManager.getUUID(); |
||||
//文件夹名称的格式是: "FocusPoint" + 大版本号 + 小版本号 + uuid + bbsUserName + randomUuid，均以下划线分隔
|
||||
StringBuilder sb = new StringBuilder(); |
||||
sb.append(FOCUS_POINT).append(SEPARATOR). |
||||
append(ProductConstants.MAIN_VERSION).append(SEPARATOR). |
||||
append(ProductConstants.MINOR_VERSION).append(SEPARATOR). |
||||
append(uuid).append(SEPARATOR).append(bbsUserName).append(SEPARATOR). |
||||
append(UUID.randomUUID()); |
||||
String fileName = String.valueOf(UUID.randomUUID()); |
||||
String pathName = StableUtils.pathJoin(ProductConstants.getEnvHome(), sb.toString(), fileName); |
||||
String folderName = StableUtils.pathJoin(ProductConstants.getEnvHome(), sb.toString()); |
||||
setFileEntityBuilder(new FileEntityBuilder(fileName, pathName, folderName)); |
||||
} |
||||
} |
||||
@ -0,0 +1,63 @@
|
||||
package com.fr.design.mainframe.messagecollect.utils; |
||||
|
||||
import com.fr.base.FRContext; |
||||
import com.fr.general.DateUtils; |
||||
import com.fr.general.IOUtils; |
||||
import com.fr.log.FineLoggerFactory; |
||||
import com.fr.stable.EncodeConstants; |
||||
import com.fr.stable.StringUtils; |
||||
import com.fr.stable.xml.XMLReadable; |
||||
import com.fr.stable.xml.XMLableReader; |
||||
import com.fr.third.javax.xml.stream.XMLStreamException; |
||||
|
||||
import java.io.ByteArrayInputStream; |
||||
import java.io.File; |
||||
import java.io.FileInputStream; |
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.io.InputStreamReader; |
||||
import java.text.DateFormat; |
||||
import java.util.Date; |
||||
|
||||
/** |
||||
* @author alex sung |
||||
* @date 2019/3/26 |
||||
*/ |
||||
public class MessageCollectUtils { |
||||
|
||||
public static void readXMLFile(XMLReadable xmlReadable, File xmlFile) { |
||||
if (xmlFile == null || !xmlFile.exists()) { |
||||
return; |
||||
} |
||||
String charset = EncodeConstants.ENCODING_UTF_8; |
||||
try { |
||||
InputStream is = new FileInputStream(xmlFile); |
||||
String fileContent = IOUtils.inputStream2String(is); |
||||
InputStream xmlInputStream = new ByteArrayInputStream(fileContent.getBytes(charset)); |
||||
InputStreamReader inputStreamReader = new InputStreamReader(xmlInputStream, charset); |
||||
XMLableReader xmlReader = XMLableReader.createXMLableReader(inputStreamReader); |
||||
if (xmlReader != null) { |
||||
xmlReader.readXMLObject(xmlReadable); |
||||
} |
||||
xmlInputStream.close(); |
||||
} catch (IOException | XMLStreamException e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||
} |
||||
} |
||||
|
||||
public static long getLastTimeMillis(String lastTime) { |
||||
if (StringUtils.isEmpty(lastTime)) { |
||||
return 0; |
||||
} |
||||
try { |
||||
return DateUtils.string2Date(lastTime, true).getTime(); |
||||
} catch (Exception e) { |
||||
return -1; |
||||
} |
||||
} |
||||
|
||||
public static String dateToString(){ |
||||
DateFormat df = FRContext.getDefaultValues().getDateTimeFormat(); |
||||
return df.format(new Date()); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue