forked from fanruan/design
5 changed files with 439 additions and 255 deletions
@ -0,0 +1,30 @@
|
||||
package com.fr.design.mainframe.messagecollect; |
||||
|
||||
import com.fr.stable.query.data.DataList; |
||||
|
||||
/** |
||||
* @author alex sung |
||||
* @date 2019/3/22 |
||||
*/ |
||||
public interface SendDataToCloudProvider { |
||||
|
||||
/** |
||||
* 获取要回传的数据 |
||||
* @param currentTime 当前时间 |
||||
* @param lastTime 上次回传时间 |
||||
* @param tClass 埋点对象类型 |
||||
* @throws Exception 取数过程中可能的异常 |
||||
*/ |
||||
<T> void getData(long currentTime, long lastTime, Class<T> tClass) throws Exception; |
||||
|
||||
/** |
||||
* @param points 从swift获取的埋点数据 |
||||
* @throws Exception 解析或存储临时文件时可能的异常 |
||||
*/ |
||||
<T> void dealWithData(DataList<T> points) throws Exception; |
||||
|
||||
/** |
||||
* 回传Zip到云中心 |
||||
*/ |
||||
void sendToCloudCenter(); |
||||
} |
@ -0,0 +1,247 @@
|
||||
package com.fr.design.mainframe.messagecollect.impl; |
||||
|
||||
import com.fr.design.mainframe.messagecollect.SendDataToCloudProvider; |
||||
import com.fr.general.CloudCenter; |
||||
import com.fr.general.IOUtils; |
||||
import com.fr.general.http.HttpRequestType; |
||||
import com.fr.general.http.HttpToolbox; |
||||
import com.fr.intelli.record.MetricRegistry; |
||||
import com.fr.json.JSONException; |
||||
import com.fr.json.JSONObject; |
||||
import com.fr.log.FineLoggerFactory; |
||||
import com.fr.stable.CommonUtils; |
||||
import com.fr.stable.EncodeConstants; |
||||
import com.fr.stable.ProductConstants; |
||||
import com.fr.stable.StableUtils; |
||||
import com.fr.stable.query.QueryFactory; |
||||
import com.fr.stable.query.condition.QueryCondition; |
||||
import com.fr.stable.query.data.DataList; |
||||
import com.fr.stable.query.restriction.RestrictionFactory; |
||||
import com.fr.stable.xml.XMLTools; |
||||
import com.fr.stable.xml.XMLable; |
||||
import com.fr.third.org.apache.http.entity.mime.MultipartEntityBuilder; |
||||
import com.fr.third.org.apache.http.entity.mime.content.FileBody; |
||||
|
||||
import java.io.ByteArrayInputStream; |
||||
import java.io.File; |
||||
import java.io.FileInputStream; |
||||
import java.io.FileOutputStream; |
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.nio.charset.Charset; |
||||
import java.util.HashMap; |
||||
import java.util.Map; |
||||
import java.util.zip.ZipEntry; |
||||
|
||||
/** |
||||
* @author alex sung |
||||
* @date 2019/3/22 |
||||
*/ |
||||
public abstract class AbstractSendDataToCloud implements SendDataToCloudProvider, XMLable { |
||||
|
||||
private static final String INTELLI_OPERATION_URL = "intelli.operation.url"; |
||||
private static final String OPERATION_URL = "https://cloud.fanruan.com/config/protect/operation"; |
||||
private static final String ATTR_SIGNATURE = "signature"; |
||||
private static final String ATTR_KEY = "key"; |
||||
private static final String FILE_NAME = "messagecollect.info"; |
||||
private static final String COLUMN_TIME = "time"; |
||||
|
||||
protected String lastTime; |
||||
private static final int PAGE_SIZE = 200; |
||||
private long totalCount = -1; |
||||
protected String fileName; |
||||
protected String pathName; |
||||
protected String folderName; |
||||
|
||||
public String getFolderName() { |
||||
return folderName; |
||||
} |
||||
|
||||
public void setFolderName(String folderName) { |
||||
this.folderName = folderName; |
||||
} |
||||
|
||||
public String getFileName() { |
||||
return fileName; |
||||
} |
||||
|
||||
public void setFileName(String fileName) { |
||||
this.fileName = fileName; |
||||
} |
||||
|
||||
public String getPathName() { |
||||
return pathName; |
||||
} |
||||
|
||||
public void setPathName(String pathName) { |
||||
this.pathName = pathName; |
||||
} |
||||
|
||||
public String getLastTime() { |
||||
return lastTime; |
||||
} |
||||
|
||||
public void setLastTime(String lastTime) { |
||||
this.lastTime = lastTime; |
||||
} |
||||
|
||||
public void saveLastTime() { |
||||
try { |
||||
FileOutputStream out = new FileOutputStream(getLastTimeFile()); |
||||
XMLTools.writeOutputStreamXML(this, out); |
||||
} catch (Exception ex) { |
||||
FineLoggerFactory.getLogger().error(ex.getMessage()); |
||||
} |
||||
} |
||||
|
||||
public static File getLastTimeFile() { |
||||
return new File(StableUtils.pathJoin(ProductConstants.getEnvHome(), FILE_NAME)); |
||||
} |
||||
|
||||
@Override |
||||
public <T> void getData(long currentTime, long lastTime, Class<T> tClass) { |
||||
queryAndSendOnePageFunctionContent(currentTime, lastTime, 0, tClass); |
||||
long page = (totalCount / PAGE_SIZE) + 1; |
||||
for (int i = 1; i < page; i++) { |
||||
queryAndSendOnePageFunctionContent(currentTime, lastTime, i, tClass); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public Object clone() throws CloneNotSupportedException { |
||||
return super.clone(); |
||||
} |
||||
|
||||
private <T> void queryAndSendOnePageFunctionContent(long current, long last, int page, Class<T> tClass) { |
||||
QueryCondition condition = QueryFactory.create() |
||||
.skip(page * PAGE_SIZE) |
||||
.count(PAGE_SIZE) |
||||
.addSort(COLUMN_TIME, true) |
||||
.addRestriction(RestrictionFactory.lte(COLUMN_TIME, current)) |
||||
.addRestriction(RestrictionFactory.gte(COLUMN_TIME, last)); |
||||
try { |
||||
DataList<T> points = MetricRegistry.getMetric().find(tClass, condition); |
||||
//第一次查询获取总记录数
|
||||
if (page == 0) { |
||||
totalCount = points.getTotalCount(); |
||||
} |
||||
dealWithData(points); |
||||
|
||||
} catch (Exception e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public <T> void dealWithData(DataList<T> tDataList) throws Exception { |
||||
generateThisPageFile(tDataList); |
||||
} |
||||
|
||||
private <T> void generateThisPageFile(DataList<T> points) { |
||||
File file = null; |
||||
try { |
||||
JSONObject jsonObject = dealWithSendFunctionContent(points); |
||||
//生成json文件
|
||||
generateFile(jsonObject); |
||||
} catch (Exception e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||
} |
||||
} |
||||
|
||||
public abstract <T> JSONObject dealWithSendFunctionContent(DataList<T> focusPoints); |
||||
|
||||
/** |
||||
* 生成zip并发送zip文件 |
||||
* @param pathName zip文件路径 |
||||
*/ |
||||
protected void sendZipFile(String pathName) { |
||||
|
||||
File file = null; |
||||
try { |
||||
file = generateZipFile(pathName); |
||||
if (file != null) { |
||||
uploadFile(file, file.getName()); |
||||
} |
||||
} catch (Exception e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||
return; |
||||
} |
||||
deleteFileAndZipFile(file, pathName); |
||||
} |
||||
|
||||
private File generateZipFile(String pathName) { |
||||
File zipFile = null; |
||||
try { |
||||
File file = new File(pathName); |
||||
zipFile = new File(pathName + ".zip"); |
||||
InputStream input = null; |
||||
java.util.zip.ZipOutputStream zipOut = null; |
||||
zipOut = new java.util.zip.ZipOutputStream(new FileOutputStream(zipFile)); |
||||
int temp = 0; |
||||
if (file.isDirectory()) { |
||||
File lists[] = file.listFiles(); |
||||
for (int i = 0; i < lists.length; i++) { |
||||
input = new FileInputStream(lists[i]); |
||||
zipOut.putNextEntry(new ZipEntry(file.getName() |
||||
+ File.separator + lists[i].getName())); |
||||
while ((temp = input.read()) != -1) { |
||||
zipOut.write(temp); |
||||
} |
||||
input.close(); |
||||
} |
||||
} |
||||
zipOut.close(); |
||||
} catch (Exception e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||
} |
||||
return zipFile; |
||||
} |
||||
|
||||
private void generateFile(JSONObject jsonObject) { |
||||
try { |
||||
String content = jsonObject.toString(); |
||||
File file = new File(pathName + ".json"); |
||||
StableUtils.makesureFileExist(file); |
||||
FileOutputStream out = new FileOutputStream(file); |
||||
InputStream in = new ByteArrayInputStream(content.getBytes(EncodeConstants.ENCODING_UTF_8)); |
||||
IOUtils.copyBinaryTo(in, out); |
||||
out.close(); |
||||
} catch (Exception e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||
} |
||||
} |
||||
|
||||
private static void uploadFile(File file, String keyFileName) throws IOException { |
||||
String url = generateSignedUploadUrl(keyFileName); |
||||
MultipartEntityBuilder builder = MultipartEntityBuilder.create() |
||||
.addPart("file", new FileBody(file)); |
||||
Map<String, String> headers = new HashMap<String, String>(); |
||||
headers.put("Content-Type", "application/zip"); |
||||
HttpToolbox.upload(url, builder, Charset.forName("utf-8"), headers, HttpRequestType.PUT); |
||||
} |
||||
|
||||
|
||||
|
||||
private void deleteFileAndZipFile(File zipFile, String pathName) { |
||||
File file = new File(StableUtils.pathJoin(ProductConstants.getEnvHome(), pathName)); |
||||
CommonUtils.deleteFile(file); |
||||
CommonUtils.deleteFile(zipFile); |
||||
} |
||||
|
||||
private static String generateSignedUploadUrl(String fileKeyName) throws IOException { |
||||
String url = CloudCenter.getInstance().acquireUrlByKind(INTELLI_OPERATION_URL, OPERATION_URL); |
||||
Map<String, String> parameters = new HashMap<String, String>(); |
||||
parameters.put(ATTR_KEY, fileKeyName); |
||||
parameters.put(ATTR_SIGNATURE, String.valueOf(CommonUtils.signature())); |
||||
String responseText = HttpToolbox.get(url, parameters); |
||||
try { |
||||
JSONObject data = new JSONObject(responseText); |
||||
if ("success".equals(data.optString("status"))) { |
||||
return data.optString("url"); |
||||
} |
||||
} catch (JSONException e) { |
||||
FineLoggerFactory.getLogger().error("Illegal response text."); |
||||
} |
||||
return null; |
||||
} |
||||
} |
@ -0,0 +1,86 @@
|
||||
package com.fr.design.mainframe.messagecollect.impl; |
||||
|
||||
import com.fr.config.MarketConfig; |
||||
import com.fr.design.DesignerEnvManager; |
||||
import com.fr.design.mainframe.messagecollect.utils.MessageCollectUtils; |
||||
import com.fr.intelli.record.FocusPoint; |
||||
import com.fr.json.JSONObject; |
||||
import com.fr.log.FineLoggerFactory; |
||||
import com.fr.stable.ProductConstants; |
||||
import com.fr.stable.StableUtils; |
||||
import com.fr.stable.query.data.DataList; |
||||
import com.fr.stable.xml.XMLPrintWriter; |
||||
import com.fr.stable.xml.XMLableReader; |
||||
|
||||
import java.util.Date; |
||||
import java.util.UUID; |
||||
|
||||
/** |
||||
* @author alex sung |
||||
* @date 2019/3/22 |
||||
*/ |
||||
public class FocusPointMessageUploader extends AbstractSendDataToCloud { |
||||
|
||||
private static final String TAG = "FocusPointMessageTag"; |
||||
private static final String SEPARATOR = "_"; |
||||
private static final String FOCUS_POINT_VERSION = "FocusPoint1003_"; |
||||
private static volatile FocusPointMessageUploader instance; |
||||
|
||||
public static FocusPointMessageUploader getInstance() { |
||||
if (instance == null) { |
||||
synchronized (FocusPointMessageUploader.class) { |
||||
if (instance == null) { |
||||
instance = new FocusPointMessageUploader(); |
||||
} |
||||
} |
||||
} |
||||
return instance; |
||||
} |
||||
|
||||
@Override |
||||
public <T> JSONObject dealWithSendFunctionContent(DataList<T> focusPoints) { |
||||
return new JSONObject(); |
||||
} |
||||
|
||||
@Override |
||||
public void sendToCloudCenter() { |
||||
MessageCollectUtils.readXMLFile(instance, getLastTimeFile()); |
||||
long currentTime = new Date().getTime(); |
||||
long lastTIme = MessageCollectUtils.getLastTimeMillis(lastTime); |
||||
try { |
||||
generatePath(); |
||||
getData(currentTime, lastTIme, FocusPoint.class); |
||||
} catch (Exception e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage()); |
||||
} |
||||
sendZipFile(getFolderName()); |
||||
saveLastTime(); |
||||
} |
||||
|
||||
@Override |
||||
public void readXML(XMLableReader reader) { |
||||
if (reader.isAttr()) { |
||||
this.setLastTime(reader.getAttrAsString("focusPointLastTime", null)); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public void writeXML(XMLPrintWriter writer) { |
||||
writer.startTAG(TAG); |
||||
writer.attr("focusPointLastTime", lastTime); |
||||
writer.end(); |
||||
} |
||||
|
||||
private void generatePath(){ |
||||
DesignerEnvManager envManager = DesignerEnvManager.getEnvManager(); |
||||
String bbsUserName = MarketConfig.getInstance().getBbsUsername(); |
||||
String uuid = envManager.getUUID(); |
||||
//文件夹名称的格式是: "FocusPoint1003_" + uuid_bbsUserName_randomUuid,均以下划线分隔
|
||||
StringBuilder sb = new StringBuilder(); |
||||
sb.append(FOCUS_POINT_VERSION).append(uuid).append(SEPARATOR).append(bbsUserName).append(SEPARATOR).append(UUID.randomUUID()); |
||||
|
||||
setFileName(String.valueOf(UUID.randomUUID())); |
||||
setPathName(StableUtils.pathJoin(ProductConstants.getEnvHome(), sb.toString(), getFileName())); |
||||
setFolderName(StableUtils.pathJoin(ProductConstants.getEnvHome(), sb.toString())); |
||||
} |
||||
} |
@ -0,0 +1,69 @@
|
||||
package com.fr.design.mainframe.messagecollect.utils; |
||||
|
||||
import com.fr.base.FRContext; |
||||
import com.fr.general.DateUtils; |
||||
import com.fr.general.IOUtils; |
||||
import com.fr.log.FineLoggerFactory; |
||||
import com.fr.stable.EncodeConstants; |
||||
import com.fr.stable.StringUtils; |
||||
import com.fr.stable.xml.XMLReadable; |
||||
import com.fr.stable.xml.XMLableReader; |
||||
import com.fr.third.javax.xml.stream.XMLStreamException; |
||||
|
||||
import java.io.ByteArrayInputStream; |
||||
import java.io.File; |
||||
import java.io.FileInputStream; |
||||
import java.io.FileNotFoundException; |
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.io.InputStreamReader; |
||||
import java.io.UnsupportedEncodingException; |
||||
import java.text.DateFormat; |
||||
import java.util.Date; |
||||
|
||||
/** |
||||
* @author alex sung |
||||
* @date 2019/3/26 |
||||
*/ |
||||
public class MessageCollectUtils { |
||||
|
||||
public static String getFileContent(File xmlFile) throws FileNotFoundException, UnsupportedEncodingException { |
||||
InputStream is = new FileInputStream(xmlFile); |
||||
return IOUtils.inputStream2String(is); |
||||
} |
||||
|
||||
public static void readXMLFile(XMLReadable xmlReadable, File xmlFile) { |
||||
if (xmlFile == null || !xmlFile.exists()) { |
||||
return; |
||||
} |
||||
String charset = EncodeConstants.ENCODING_UTF_8; |
||||
try { |
||||
String fileContent = MessageCollectUtils.getFileContent(xmlFile); |
||||
InputStream xmlInputStream = new ByteArrayInputStream(fileContent.getBytes(charset)); |
||||
InputStreamReader inputStreamReader = new InputStreamReader(xmlInputStream, charset); |
||||
XMLableReader xmlReader = XMLableReader.createXMLableReader(inputStreamReader); |
||||
if (xmlReader != null) { |
||||
xmlReader.readXMLObject(xmlReadable); |
||||
} |
||||
xmlInputStream.close(); |
||||
} catch (IOException | XMLStreamException e) { |
||||
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||
} |
||||
} |
||||
|
||||
public static String newDateToString() { |
||||
DateFormat df = FRContext.getDefaultValues().getDateTimeFormat(); |
||||
return df.format(new Date()); |
||||
} |
||||
|
||||
public static long getLastTimeMillis(String lastTime) { |
||||
if (StringUtils.isEmpty(lastTime)) { |
||||
return 0; |
||||
} |
||||
try { |
||||
return DateUtils.string2Date(lastTime, true).getTime(); |
||||
} catch (Exception e) { |
||||
return -1; |
||||
} |
||||
} |
||||
} |
Loading…
Reference in new issue