forked from fanruan/design
5 changed files with 439 additions and 255 deletions
@ -0,0 +1,30 @@ |
|||||||
|
package com.fr.design.mainframe.messagecollect; |
||||||
|
|
||||||
|
import com.fr.stable.query.data.DataList; |
||||||
|
|
||||||
|
/** |
||||||
|
* @author alex sung |
||||||
|
* @date 2019/3/22 |
||||||
|
*/ |
||||||
|
public interface SendDataToCloudProvider { |
||||||
|
|
||||||
|
/** |
||||||
|
* 获取要回传的数据 |
||||||
|
* @param currentTime 当前时间 |
||||||
|
* @param lastTime 上次回传时间 |
||||||
|
* @param tClass 埋点对象类型 |
||||||
|
* @throws Exception 取数过程中可能的异常 |
||||||
|
*/ |
||||||
|
<T> void getData(long currentTime, long lastTime, Class<T> tClass) throws Exception; |
||||||
|
|
||||||
|
/** |
||||||
|
* @param points 从swift获取的埋点数据 |
||||||
|
* @throws Exception 解析或存储临时文件时可能的异常 |
||||||
|
*/ |
||||||
|
<T> void dealWithData(DataList<T> points) throws Exception; |
||||||
|
|
||||||
|
/** |
||||||
|
* 回传Zip到云中心 |
||||||
|
*/ |
||||||
|
void sendToCloudCenter(); |
||||||
|
} |
@ -0,0 +1,247 @@ |
|||||||
|
package com.fr.design.mainframe.messagecollect.impl; |
||||||
|
|
||||||
|
import com.fr.design.mainframe.messagecollect.SendDataToCloudProvider; |
||||||
|
import com.fr.general.CloudCenter; |
||||||
|
import com.fr.general.IOUtils; |
||||||
|
import com.fr.general.http.HttpRequestType; |
||||||
|
import com.fr.general.http.HttpToolbox; |
||||||
|
import com.fr.intelli.record.MetricRegistry; |
||||||
|
import com.fr.json.JSONException; |
||||||
|
import com.fr.json.JSONObject; |
||||||
|
import com.fr.log.FineLoggerFactory; |
||||||
|
import com.fr.stable.CommonUtils; |
||||||
|
import com.fr.stable.EncodeConstants; |
||||||
|
import com.fr.stable.ProductConstants; |
||||||
|
import com.fr.stable.StableUtils; |
||||||
|
import com.fr.stable.query.QueryFactory; |
||||||
|
import com.fr.stable.query.condition.QueryCondition; |
||||||
|
import com.fr.stable.query.data.DataList; |
||||||
|
import com.fr.stable.query.restriction.RestrictionFactory; |
||||||
|
import com.fr.stable.xml.XMLTools; |
||||||
|
import com.fr.stable.xml.XMLable; |
||||||
|
import com.fr.third.org.apache.http.entity.mime.MultipartEntityBuilder; |
||||||
|
import com.fr.third.org.apache.http.entity.mime.content.FileBody; |
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream; |
||||||
|
import java.io.File; |
||||||
|
import java.io.FileInputStream; |
||||||
|
import java.io.FileOutputStream; |
||||||
|
import java.io.IOException; |
||||||
|
import java.io.InputStream; |
||||||
|
import java.nio.charset.Charset; |
||||||
|
import java.util.HashMap; |
||||||
|
import java.util.Map; |
||||||
|
import java.util.zip.ZipEntry; |
||||||
|
|
||||||
|
/** |
||||||
|
* @author alex sung |
||||||
|
* @date 2019/3/22 |
||||||
|
*/ |
||||||
|
public abstract class AbstractSendDataToCloud implements SendDataToCloudProvider, XMLable { |
||||||
|
|
||||||
|
private static final String INTELLI_OPERATION_URL = "intelli.operation.url"; |
||||||
|
private static final String OPERATION_URL = "https://cloud.fanruan.com/config/protect/operation"; |
||||||
|
private static final String ATTR_SIGNATURE = "signature"; |
||||||
|
private static final String ATTR_KEY = "key"; |
||||||
|
private static final String FILE_NAME = "messagecollect.info"; |
||||||
|
private static final String COLUMN_TIME = "time"; |
||||||
|
|
||||||
|
protected String lastTime; |
||||||
|
private static final int PAGE_SIZE = 200; |
||||||
|
private long totalCount = -1; |
||||||
|
protected String fileName; |
||||||
|
protected String pathName; |
||||||
|
protected String folderName; |
||||||
|
|
||||||
|
public String getFolderName() { |
||||||
|
return folderName; |
||||||
|
} |
||||||
|
|
||||||
|
public void setFolderName(String folderName) { |
||||||
|
this.folderName = folderName; |
||||||
|
} |
||||||
|
|
||||||
|
public String getFileName() { |
||||||
|
return fileName; |
||||||
|
} |
||||||
|
|
||||||
|
public void setFileName(String fileName) { |
||||||
|
this.fileName = fileName; |
||||||
|
} |
||||||
|
|
||||||
|
public String getPathName() { |
||||||
|
return pathName; |
||||||
|
} |
||||||
|
|
||||||
|
public void setPathName(String pathName) { |
||||||
|
this.pathName = pathName; |
||||||
|
} |
||||||
|
|
||||||
|
public String getLastTime() { |
||||||
|
return lastTime; |
||||||
|
} |
||||||
|
|
||||||
|
public void setLastTime(String lastTime) { |
||||||
|
this.lastTime = lastTime; |
||||||
|
} |
||||||
|
|
||||||
|
public void saveLastTime() { |
||||||
|
try { |
||||||
|
FileOutputStream out = new FileOutputStream(getLastTimeFile()); |
||||||
|
XMLTools.writeOutputStreamXML(this, out); |
||||||
|
} catch (Exception ex) { |
||||||
|
FineLoggerFactory.getLogger().error(ex.getMessage()); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public static File getLastTimeFile() { |
||||||
|
return new File(StableUtils.pathJoin(ProductConstants.getEnvHome(), FILE_NAME)); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public <T> void getData(long currentTime, long lastTime, Class<T> tClass) { |
||||||
|
queryAndSendOnePageFunctionContent(currentTime, lastTime, 0, tClass); |
||||||
|
long page = (totalCount / PAGE_SIZE) + 1; |
||||||
|
for (int i = 1; i < page; i++) { |
||||||
|
queryAndSendOnePageFunctionContent(currentTime, lastTime, i, tClass); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Object clone() throws CloneNotSupportedException { |
||||||
|
return super.clone(); |
||||||
|
} |
||||||
|
|
||||||
|
private <T> void queryAndSendOnePageFunctionContent(long current, long last, int page, Class<T> tClass) { |
||||||
|
QueryCondition condition = QueryFactory.create() |
||||||
|
.skip(page * PAGE_SIZE) |
||||||
|
.count(PAGE_SIZE) |
||||||
|
.addSort(COLUMN_TIME, true) |
||||||
|
.addRestriction(RestrictionFactory.lte(COLUMN_TIME, current)) |
||||||
|
.addRestriction(RestrictionFactory.gte(COLUMN_TIME, last)); |
||||||
|
try { |
||||||
|
DataList<T> points = MetricRegistry.getMetric().find(tClass, condition); |
||||||
|
//第一次查询获取总记录数
|
||||||
|
if (page == 0) { |
||||||
|
totalCount = points.getTotalCount(); |
||||||
|
} |
||||||
|
dealWithData(points); |
||||||
|
|
||||||
|
} catch (Exception e) { |
||||||
|
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public <T> void dealWithData(DataList<T> tDataList) throws Exception { |
||||||
|
generateThisPageFile(tDataList); |
||||||
|
} |
||||||
|
|
||||||
|
private <T> void generateThisPageFile(DataList<T> points) { |
||||||
|
File file = null; |
||||||
|
try { |
||||||
|
JSONObject jsonObject = dealWithSendFunctionContent(points); |
||||||
|
//生成json文件
|
||||||
|
generateFile(jsonObject); |
||||||
|
} catch (Exception e) { |
||||||
|
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public abstract <T> JSONObject dealWithSendFunctionContent(DataList<T> focusPoints); |
||||||
|
|
||||||
|
/** |
||||||
|
* 生成zip并发送zip文件 |
||||||
|
* @param pathName zip文件路径 |
||||||
|
*/ |
||||||
|
protected void sendZipFile(String pathName) { |
||||||
|
|
||||||
|
File file = null; |
||||||
|
try { |
||||||
|
file = generateZipFile(pathName); |
||||||
|
if (file != null) { |
||||||
|
uploadFile(file, file.getName()); |
||||||
|
} |
||||||
|
} catch (Exception e) { |
||||||
|
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||||
|
return; |
||||||
|
} |
||||||
|
deleteFileAndZipFile(file, pathName); |
||||||
|
} |
||||||
|
|
||||||
|
private File generateZipFile(String pathName) { |
||||||
|
File zipFile = null; |
||||||
|
try { |
||||||
|
File file = new File(pathName); |
||||||
|
zipFile = new File(pathName + ".zip"); |
||||||
|
InputStream input = null; |
||||||
|
java.util.zip.ZipOutputStream zipOut = null; |
||||||
|
zipOut = new java.util.zip.ZipOutputStream(new FileOutputStream(zipFile)); |
||||||
|
int temp = 0; |
||||||
|
if (file.isDirectory()) { |
||||||
|
File lists[] = file.listFiles(); |
||||||
|
for (int i = 0; i < lists.length; i++) { |
||||||
|
input = new FileInputStream(lists[i]); |
||||||
|
zipOut.putNextEntry(new ZipEntry(file.getName() |
||||||
|
+ File.separator + lists[i].getName())); |
||||||
|
while ((temp = input.read()) != -1) { |
||||||
|
zipOut.write(temp); |
||||||
|
} |
||||||
|
input.close(); |
||||||
|
} |
||||||
|
} |
||||||
|
zipOut.close(); |
||||||
|
} catch (Exception e) { |
||||||
|
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||||
|
} |
||||||
|
return zipFile; |
||||||
|
} |
||||||
|
|
||||||
|
private void generateFile(JSONObject jsonObject) { |
||||||
|
try { |
||||||
|
String content = jsonObject.toString(); |
||||||
|
File file = new File(pathName + ".json"); |
||||||
|
StableUtils.makesureFileExist(file); |
||||||
|
FileOutputStream out = new FileOutputStream(file); |
||||||
|
InputStream in = new ByteArrayInputStream(content.getBytes(EncodeConstants.ENCODING_UTF_8)); |
||||||
|
IOUtils.copyBinaryTo(in, out); |
||||||
|
out.close(); |
||||||
|
} catch (Exception e) { |
||||||
|
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static void uploadFile(File file, String keyFileName) throws IOException { |
||||||
|
String url = generateSignedUploadUrl(keyFileName); |
||||||
|
MultipartEntityBuilder builder = MultipartEntityBuilder.create() |
||||||
|
.addPart("file", new FileBody(file)); |
||||||
|
Map<String, String> headers = new HashMap<String, String>(); |
||||||
|
headers.put("Content-Type", "application/zip"); |
||||||
|
HttpToolbox.upload(url, builder, Charset.forName("utf-8"), headers, HttpRequestType.PUT); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
private void deleteFileAndZipFile(File zipFile, String pathName) { |
||||||
|
File file = new File(StableUtils.pathJoin(ProductConstants.getEnvHome(), pathName)); |
||||||
|
CommonUtils.deleteFile(file); |
||||||
|
CommonUtils.deleteFile(zipFile); |
||||||
|
} |
||||||
|
|
||||||
|
private static String generateSignedUploadUrl(String fileKeyName) throws IOException { |
||||||
|
String url = CloudCenter.getInstance().acquireUrlByKind(INTELLI_OPERATION_URL, OPERATION_URL); |
||||||
|
Map<String, String> parameters = new HashMap<String, String>(); |
||||||
|
parameters.put(ATTR_KEY, fileKeyName); |
||||||
|
parameters.put(ATTR_SIGNATURE, String.valueOf(CommonUtils.signature())); |
||||||
|
String responseText = HttpToolbox.get(url, parameters); |
||||||
|
try { |
||||||
|
JSONObject data = new JSONObject(responseText); |
||||||
|
if ("success".equals(data.optString("status"))) { |
||||||
|
return data.optString("url"); |
||||||
|
} |
||||||
|
} catch (JSONException e) { |
||||||
|
FineLoggerFactory.getLogger().error("Illegal response text."); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,86 @@ |
|||||||
|
package com.fr.design.mainframe.messagecollect.impl; |
||||||
|
|
||||||
|
import com.fr.config.MarketConfig; |
||||||
|
import com.fr.design.DesignerEnvManager; |
||||||
|
import com.fr.design.mainframe.messagecollect.utils.MessageCollectUtils; |
||||||
|
import com.fr.intelli.record.FocusPoint; |
||||||
|
import com.fr.json.JSONObject; |
||||||
|
import com.fr.log.FineLoggerFactory; |
||||||
|
import com.fr.stable.ProductConstants; |
||||||
|
import com.fr.stable.StableUtils; |
||||||
|
import com.fr.stable.query.data.DataList; |
||||||
|
import com.fr.stable.xml.XMLPrintWriter; |
||||||
|
import com.fr.stable.xml.XMLableReader; |
||||||
|
|
||||||
|
import java.util.Date; |
||||||
|
import java.util.UUID; |
||||||
|
|
||||||
|
/** |
||||||
|
* @author alex sung |
||||||
|
* @date 2019/3/22 |
||||||
|
*/ |
||||||
|
public class FocusPointMessageUploader extends AbstractSendDataToCloud { |
||||||
|
|
||||||
|
private static final String TAG = "FocusPointMessageTag"; |
||||||
|
private static final String SEPARATOR = "_"; |
||||||
|
private static final String FOCUS_POINT_VERSION = "FocusPoint1003_"; |
||||||
|
private static volatile FocusPointMessageUploader instance; |
||||||
|
|
||||||
|
public static FocusPointMessageUploader getInstance() { |
||||||
|
if (instance == null) { |
||||||
|
synchronized (FocusPointMessageUploader.class) { |
||||||
|
if (instance == null) { |
||||||
|
instance = new FocusPointMessageUploader(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
return instance; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public <T> JSONObject dealWithSendFunctionContent(DataList<T> focusPoints) { |
||||||
|
return new JSONObject(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void sendToCloudCenter() { |
||||||
|
MessageCollectUtils.readXMLFile(instance, getLastTimeFile()); |
||||||
|
long currentTime = new Date().getTime(); |
||||||
|
long lastTIme = MessageCollectUtils.getLastTimeMillis(lastTime); |
||||||
|
try { |
||||||
|
generatePath(); |
||||||
|
getData(currentTime, lastTIme, FocusPoint.class); |
||||||
|
} catch (Exception e) { |
||||||
|
FineLoggerFactory.getLogger().error(e.getMessage()); |
||||||
|
} |
||||||
|
sendZipFile(getFolderName()); |
||||||
|
saveLastTime(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void readXML(XMLableReader reader) { |
||||||
|
if (reader.isAttr()) { |
||||||
|
this.setLastTime(reader.getAttrAsString("focusPointLastTime", null)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void writeXML(XMLPrintWriter writer) { |
||||||
|
writer.startTAG(TAG); |
||||||
|
writer.attr("focusPointLastTime", lastTime); |
||||||
|
writer.end(); |
||||||
|
} |
||||||
|
|
||||||
|
private void generatePath(){ |
||||||
|
DesignerEnvManager envManager = DesignerEnvManager.getEnvManager(); |
||||||
|
String bbsUserName = MarketConfig.getInstance().getBbsUsername(); |
||||||
|
String uuid = envManager.getUUID(); |
||||||
|
//文件夹名称的格式是: "FocusPoint1003_" + uuid_bbsUserName_randomUuid,均以下划线分隔
|
||||||
|
StringBuilder sb = new StringBuilder(); |
||||||
|
sb.append(FOCUS_POINT_VERSION).append(uuid).append(SEPARATOR).append(bbsUserName).append(SEPARATOR).append(UUID.randomUUID()); |
||||||
|
|
||||||
|
setFileName(String.valueOf(UUID.randomUUID())); |
||||||
|
setPathName(StableUtils.pathJoin(ProductConstants.getEnvHome(), sb.toString(), getFileName())); |
||||||
|
setFolderName(StableUtils.pathJoin(ProductConstants.getEnvHome(), sb.toString())); |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,69 @@ |
|||||||
|
package com.fr.design.mainframe.messagecollect.utils; |
||||||
|
|
||||||
|
import com.fr.base.FRContext; |
||||||
|
import com.fr.general.DateUtils; |
||||||
|
import com.fr.general.IOUtils; |
||||||
|
import com.fr.log.FineLoggerFactory; |
||||||
|
import com.fr.stable.EncodeConstants; |
||||||
|
import com.fr.stable.StringUtils; |
||||||
|
import com.fr.stable.xml.XMLReadable; |
||||||
|
import com.fr.stable.xml.XMLableReader; |
||||||
|
import com.fr.third.javax.xml.stream.XMLStreamException; |
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream; |
||||||
|
import java.io.File; |
||||||
|
import java.io.FileInputStream; |
||||||
|
import java.io.FileNotFoundException; |
||||||
|
import java.io.IOException; |
||||||
|
import java.io.InputStream; |
||||||
|
import java.io.InputStreamReader; |
||||||
|
import java.io.UnsupportedEncodingException; |
||||||
|
import java.text.DateFormat; |
||||||
|
import java.util.Date; |
||||||
|
|
||||||
|
/** |
||||||
|
* @author alex sung |
||||||
|
* @date 2019/3/26 |
||||||
|
*/ |
||||||
|
public class MessageCollectUtils { |
||||||
|
|
||||||
|
public static String getFileContent(File xmlFile) throws FileNotFoundException, UnsupportedEncodingException { |
||||||
|
InputStream is = new FileInputStream(xmlFile); |
||||||
|
return IOUtils.inputStream2String(is); |
||||||
|
} |
||||||
|
|
||||||
|
public static void readXMLFile(XMLReadable xmlReadable, File xmlFile) { |
||||||
|
if (xmlFile == null || !xmlFile.exists()) { |
||||||
|
return; |
||||||
|
} |
||||||
|
String charset = EncodeConstants.ENCODING_UTF_8; |
||||||
|
try { |
||||||
|
String fileContent = MessageCollectUtils.getFileContent(xmlFile); |
||||||
|
InputStream xmlInputStream = new ByteArrayInputStream(fileContent.getBytes(charset)); |
||||||
|
InputStreamReader inputStreamReader = new InputStreamReader(xmlInputStream, charset); |
||||||
|
XMLableReader xmlReader = XMLableReader.createXMLableReader(inputStreamReader); |
||||||
|
if (xmlReader != null) { |
||||||
|
xmlReader.readXMLObject(xmlReadable); |
||||||
|
} |
||||||
|
xmlInputStream.close(); |
||||||
|
} catch (IOException | XMLStreamException e) { |
||||||
|
FineLoggerFactory.getLogger().error(e.getMessage(), e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public static String newDateToString() { |
||||||
|
DateFormat df = FRContext.getDefaultValues().getDateTimeFormat(); |
||||||
|
return df.format(new Date()); |
||||||
|
} |
||||||
|
|
||||||
|
public static long getLastTimeMillis(String lastTime) { |
||||||
|
if (StringUtils.isEmpty(lastTime)) { |
||||||
|
return 0; |
||||||
|
} |
||||||
|
try { |
||||||
|
return DateUtils.string2Date(lastTime, true).getTime(); |
||||||
|
} catch (Exception e) { |
||||||
|
return -1; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
Loading…
Reference in new issue