398 lines
17 KiB
Java
398 lines
17 KiB
Java
package com.celnet.datadump;
|
||
|
||
import com.alibaba.fastjson.JSONObject;
|
||
import com.alibaba.fastjson2.JSON;
|
||
import com.alibaba.fastjson2.JSONArray;
|
||
import com.aliyun.oss.OSS;
|
||
import com.aliyun.oss.OSSClientBuilder;
|
||
import com.aliyun.oss.model.PutObjectResult;
|
||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||
import com.celnet.datadump.entity.DataBatch;
|
||
import com.celnet.datadump.entity.DataObject;
|
||
import com.celnet.datadump.global.Const;
|
||
import com.celnet.datadump.mapper.CustomMapper;
|
||
import com.celnet.datadump.service.CommonService;
|
||
import com.celnet.datadump.service.DataBatchService;
|
||
import com.celnet.datadump.service.DataFieldService;
|
||
import com.celnet.datadump.service.DataObjectService;
|
||
import com.celnet.datadump.service.impl.DataFieldServiceImpl;
|
||
import com.celnet.datadump.util.DataUtil;
|
||
import com.google.gson.JsonObject;
|
||
import com.sforce.soap.partner.*;
|
||
import com.sforce.soap.partner.sobject.SObject;
|
||
import com.sforce.ws.ConnectorConfig;
|
||
import com.sforce.ws.util.Base64;
|
||
import okhttp3.*;
|
||
import org.apache.commons.io.FileUtils;
|
||
import org.apache.commons.lang3.time.DateFormatUtils;
|
||
import org.apache.commons.lang3.time.DateUtils;
|
||
import org.apache.http.HttpEntity;
|
||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||
import org.apache.http.client.methods.HttpPost;
|
||
import org.apache.http.entity.ContentType;
|
||
import org.apache.http.entity.mime.HttpMultipartMode;
|
||
import org.apache.http.entity.mime.MultipartEntityBuilder;
|
||
import org.apache.http.entity.mime.content.FileBody;
|
||
import org.apache.http.impl.client.CloseableHttpClient;
|
||
import org.apache.http.impl.client.HttpClients;
|
||
import org.apache.http.util.EntityUtils;
|
||
import org.junit.jupiter.api.Test;
|
||
import org.springframework.beans.factory.annotation.Autowired;
|
||
import org.springframework.boot.test.context.SpringBootTest;
|
||
import org.springframework.http.ContentDisposition;
|
||
import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
|
||
import org.springframework.test.context.junit4.SpringRunner;
|
||
|
||
import java.io.File;
|
||
import java.io.FileInputStream;
|
||
import java.io.InputStream;
|
||
import java.nio.charset.StandardCharsets;
|
||
import java.sql.Blob;
|
||
import java.text.SimpleDateFormat;
|
||
import java.time.LocalDateTime;
|
||
import java.util.*;
|
||
import java.util.concurrent.LinkedBlockingQueue;
|
||
import java.util.concurrent.ThreadPoolExecutor;
|
||
import java.util.concurrent.TimeUnit;
|
||
|
||
|
||
/**
|
||
* @author Red
|
||
* @description
|
||
* @date 2022/09/23
|
||
*/
|
||
public class DataDumpApplicationTests {
|
||
public static final String SF_DATE_FORMAT2 = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
|
||
|
||
@Autowired
|
||
private DataFieldService dataFieldService;
|
||
|
||
@Autowired
|
||
private DataBatchService dataBatchService;
|
||
|
||
@Autowired
|
||
private CustomMapper customMapper;
|
||
|
||
@Autowired
|
||
private CommonService commonService;
|
||
|
||
@Autowired
|
||
private DataObjectService dataObjectService;
|
||
|
||
// @Test
|
||
// public void run13() {
|
||
// // yourEndpoint填写Bucket所在地域对应的Endpoint。以华东1（杭州）为例，Endpoint填写为https://oss-cn-hangzhou.aliyuncs.com。
|
||
// String endpoint = "http://crm-backup-oss.oss-cn-shenzhen.aliyuncs.com";
|
||
//// 阿里云账号AccessKey拥有所有API的访问权限，风险很高。强烈建议您创建并使用RAM用户进行API访问或日常运维，请登录RAM控制台创建RAM用户。
|
||
// String accessKeyId = "LTAI5tBEFLSJ6hccuDH7wKkv";
|
||
// String accessKeySecret = "Q3d6MjrWJCmwz1ND54hLnUCwYAWsDa";
|
||
//
|
||
//// 创建OSSClient实例。
|
||
// OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret);
|
||
//
|
||
// File doc = new File("/Users/zhuxuanfan/Downloads/a.pdf");
|
||
// PutObjectResult result = ossClient.putObject("crm-backup-oss", "a.pdf", doc);
|
||
//// 关闭OSSClient。
|
||
// ossClient.shutdown();
|
||
// }
|
||
//
|
||
// @Test
|
||
// public void run12() throws Exception {
|
||
// File file = new File("/Users/zhuxuanfan/Downloads/a.txt");
|
||
// List<String> lines = FileUtils.readLines(file, "UTF-8");
|
||
// StringBuilder sb = new StringBuilder();
|
||
// for (String line : lines) {
|
||
// sb.append(line);
|
||
// }
|
||
// byte[] bytes = sb.toString().getBytes();
|
||
// byte[] decode = Base64.decode(bytes);
|
||
//// System.out.println(sb);
|
||
// File doc = new File("/Users/zhuxuanfan/Downloads/a.pdf");
|
||
// FileUtils.writeByteArrayToFile(doc, decode);
|
||
//
|
||
// }
|
||
|
||
@Test
|
||
public void run() throws Exception {
|
||
String Username = "simone.xu@celnet.com.cn.leo.leodev2023";
|
||
String Password = "Leocrm2025";
|
||
String AuthEndpoint = "https://test.salesforce.com/services/Soap/u/56.0";
|
||
|
||
ConnectorConfig config = new ConnectorConfig();
|
||
config.setUsername(Username);
|
||
config.setPassword(Password);
|
||
config.setAuthEndpoint(AuthEndpoint);
|
||
|
||
PartnerConnection partnerConnection = new PartnerConnection(config);
|
||
System.out.println(partnerConnection.getUserInfo().getOrganizationId());
|
||
// String documentId = "069N0000001XyC7IAK";
|
||
// String contentVersionId = "0680k000001gFCjAAM";
|
||
// SObject[] accounts = new SObject[1];
|
||
// SObject account = new SObject();
|
||
// account.setType("ContentDocumentLink");
|
||
// account.setField("ContentDocumentId", "0690k000001cS6CAAU");
|
||
// account.setField("LinkedEntityId", "005N000000BaKeDIAV");
|
||
// accounts[0]=account;
|
||
//
|
||
// SaveResult[] saveResults = partnerConnection.create(accounts);
|
||
// System.out.println(JSON.toJSONString(saveResults));
|
||
|
||
// QueryResult queryResult = partnerConnection.queryAll("SELECT Id, ContentDocumentId, IsDeleted FROM ContentVersion where Id = " + "'" + contentVersionId + "'");
|
||
// DescribeSObjectResult dsr = partnerConnection.describeSObject("ContentVersion");
|
||
// Field[] dsrFields = dsr.getFields();
|
||
// SObject[] records = queryResult.getRecords();
|
||
// JSONArray objects = DataUtil.toJsonArray(records, dsrFields);
|
||
// System.out.println(JSON.toJSONString(objects));
|
||
// System.out.println(JSON.toJSONString(queryResult.getRecords()[0].getChild("LinkedEntity").getChild("Type").getValue()));
|
||
// account.setField("CreatedDate", Calendar.getInstance());
|
||
//// Map<String, Object> CreatedByIdMap = customMapper.getById("new_id", "User", data.get(j-1).get("CreatedById").toString());
|
||
// account.setField("CreatedById", "005N000000BaKBhIAN");
|
||
// account.setField("AgentRating__c", "一级代理");
|
||
// accounts[0] = account;
|
||
// SaveResult[] saveResults = partnerConnection.create(accounts);
|
||
// System.out.println(JSON.toJSONString(saveResults));
|
||
// String SobjectName = "ApexClass";
|
||
//
|
||
// DescribeGlobalResult result = partnerConnection.describeGlobal();
|
||
// DescribeGlobalSObjectResult[] Sobjects = result.getSobjects();
|
||
// List<String> sobjectList = new ArrayList<String>();
|
||
// List<String> typeList = new ArrayList<>();
|
||
// typeList.add("History");
|
||
// typeList.add("Share");
|
||
// typeList.add("Tag");
|
||
// for(DescribeGlobalSObjectResult Sobject : Sobjects) {
|
||
// if (typeList.contains(Sobject.getAssociateEntityType())) {
|
||
// continue;
|
||
// }
|
||
//
|
||
// if (Sobject.getCreateable()) {
|
||
// sobjectList.add(Sobject.getName());
|
||
// }
|
||
//
|
||
// if (Sobject.getName().equals(SobjectName)) {
|
||
// System.out.println("sobject: " + JSON.toJSONString(Sobject));
|
||
// }
|
||
// }
|
||
// System.out.println("Sobjects: " + JSON.toJSONString(sobjectList));
|
||
// System.out.println("Sobjects count num: " + JSON.toJSONString(sobjectList.size()));
|
||
// List<String> filedlist = new ArrayList<String>();
|
||
//
|
||
// DescribeSObjectResult[] dsrArray = PartnerConnection.describeSObjects(new String[]{SobjectName});
|
||
// DescribeSObjectResult dsr = dsrArray[0];
|
||
// System.out.println(JSON.toJSONString(dsr));
|
||
// for (Field field : dsr.getFields()) {
|
||
// filedlist.add(field.getName());
|
||
// }
|
||
//
|
||
// String contentDocumentId = "069N0000002IlqDIAS";
|
||
// String sql = "SELECT Id, LinkedEntityId, LinkedEntity.type, ContentDocumentId, IsDeleted, SystemModstamp, ShareType, Visibility FROM ContentDocumentLink where ContentDocumentId = '" + contentDocumentId + "'";
|
||
// QueryResult queryResult = partnerConnection.query(sql);
|
||
//
|
||
// System.out.println(JSONObject.toJSONString(queryResult.getRecords()[0].getChild("LinkedEntity").getChild("Type").getValue()));
|
||
|
||
|
||
}
|
||
//
|
||
// private static final ThreadPoolExecutor THREAD_POOL_EXECUTOR = new ThreadPoolExecutor(1,
|
||
// 1,
|
||
// 0L,
|
||
// TimeUnit.MILLISECONDS,
|
||
// new LinkedBlockingQueue<>(),
|
||
// new CustomizableThreadFactory("threadPoolExecutor-"));
|
||
//
|
||
//
|
||
// @Test
|
||
// public void test1() throws Exception {
|
||
// Date endCreateDate = new Date(), weekLastDay;
|
||
// Date startDate = DateUtils.parseDate("2016-01-01", "yyyy-MM-dd");
|
||
// do {
|
||
// weekLastDay = DataUtil.getMonthLastDay(endCreateDate, startDate);
|
||
// System.out.print("start: " + DateFormatUtils.format(startDate, "yyyy-MM-dd HH:mm:ss"));
|
||
// System.out.println(", end: " + DateFormatUtils.format(weekLastDay, "yyyy-MM-dd HH:mm:ss"));
|
||
// startDate = weekLastDay;
|
||
// } while (weekLastDay.compareTo(endCreateDate) < 0);
|
||
//
|
||
// }
|
||
//
|
||
//
|
||
// @Test
|
||
// public void test2() throws Exception {
|
||
// THREAD_POOL_EXECUTOR.submit(() -> {
|
||
// try {
|
||
// Thread.sleep(2000);
|
||
// System.out.println(1);
|
||
// } catch (InterruptedException e) {
|
||
// throw new RuntimeException(e);
|
||
// }
|
||
// });
|
||
// THREAD_POOL_EXECUTOR.submit(() -> {
|
||
// try {
|
||
// Thread.sleep(2000);
|
||
// System.out.println(2);
|
||
// } catch (InterruptedException e) {
|
||
// throw new RuntimeException(e);
|
||
// }
|
||
// });
|
||
// THREAD_POOL_EXECUTOR.purge();
|
||
// while (THREAD_POOL_EXECUTOR.getActiveCount() > 0) {
|
||
// System.out.println("ActiveCount: " + THREAD_POOL_EXECUTOR.getActiveCount() + " count:" + THREAD_POOL_EXECUTOR.getQueue().size());
|
||
// try {
|
||
// Thread.sleep(1000);
|
||
// } catch (InterruptedException e) {
|
||
// throw new RuntimeException(e);
|
||
// }
|
||
//
|
||
// }
|
||
// }
|
||
//
|
||
//
|
||
// @Test
|
||
// public void test() throws Exception {
|
||
// String Username = "guoshisen@163.com";
|
||
// String Password = "celnet2024";
|
||
// String AuthEndpoint = "https://login.salesforce.com/services/Soap/u/56.0";
|
||
//
|
||
// ConnectorConfig config = new ConnectorConfig();
|
||
// config.setUsername(Username);
|
||
// config.setPassword(Password);
|
||
// config.setAuthEndpoint(AuthEndpoint);
|
||
//
|
||
// PartnerConnection partnerConnection = new PartnerConnection(config);
|
||
// String token = partnerConnection.getSessionHeader().getSessionId();
|
||
// System.out.println(token);
|
||
//
|
||
// File file = new File("/Users/mac/Downloads/6e146b3fda74b7ad20f1669c6f3043d8.jpeg");
|
||
// // 1.创建OkHttpClient对象
|
||
// OkHttpClient httpClient = new OkHttpClient();
|
||
//
|
||
//
|
||
// // 2.创建MultipartBody.Builder，并添加表单参数和文件参数
|
||
// MultipartBody.Builder builder = new MultipartBody.Builder();
|
||
// builder.setType(MultipartBody.FORM);
|
||
// builder.addFormDataPart("entity_content", "{'Title':'文件上传test','PathOnClient':'6e146b3fda74b7ad20f1669c6f3043d8.jpeg'}");
|
||
// builder.addFormDataPart("VersionData", file.getName(), RequestBody.create(MediaType.parse("application/octet-stream"), file));
|
||
//
|
||
// // 构建RequestBody对象
|
||
// RequestBody requestBody = builder.build();
|
||
//
|
||
// // 3.创建HttpPost请求，并设置请求RequestBody和请求头，发送请求并处理响应
|
||
//// Request request = new Request.Builder()
|
||
//// .url("https://devin-dev-ed.my.salesforce.com/services/data/v56.0/sobjects/ContentVersion")
|
||
//// .post(requestBody)
|
||
//// .addHeader("Content-Type", "multipart/form-data;boundary=\"--boundary_string\"")
|
||
//// .addHeader("Authorization", "Bearer " + token)
|
||
//// .addHeader("connection", "keep-alive")
|
||
//// .build();
|
||
//// Response response = httpClient.newCall(request).execute();
|
||
//// System.out.println(JSON.toJSONString(response.body().string()));
|
||
// }
|
||
|
||
// @Test
|
||
// public void tableRelation(){
|
||
// dataFieldService.tableRelation();
|
||
// }
|
||
|
||
@Test
|
||
public void uploadFile() throws Exception {
|
||
// String Username = "guoshisen@163.com";
|
||
// String Password = "celnet2024";
|
||
// String AuthEndpoint = "https://login.salesforce.com/services/Soap/u/56.0";
|
||
|
||
String Username = "simone.xu@celnet.com.cn.leo.leodev2023";
|
||
String Password = "Leocrm2025";
|
||
String AuthEndpoint = "https://test.salesforce.com/services/Soap/u/56.0";
|
||
|
||
ConnectorConfig config = new ConnectorConfig();
|
||
config.setUsername(Username);
|
||
config.setPassword(Password);
|
||
config.setAuthEndpoint(AuthEndpoint);
|
||
|
||
PartnerConnection partnerConnection = new PartnerConnection(config);
|
||
String token = partnerConnection.getSessionHeader().getSessionId();
|
||
System.out.println(token);
|
||
|
||
File file = new File("/Users/mac/Downloads/6e146b3fda74b7ad20f1669c6f3043d8.jpeg");
|
||
String fileName = file.getName();
|
||
|
||
HttpPost httpPost = new HttpPost("https://saiyi2022--leodev2023.sandbox.my.salesforce.com/services/data/v55.0/sobjects/Attachment");
|
||
httpPost.setHeader("Authorization", "Bearer " + token);
|
||
httpPost.setHeader("connection", "keep-alive");
|
||
CloseableHttpResponse response = null;
|
||
String respContent = null;
|
||
|
||
|
||
JSONObject credentialsJsonParam = new JSONObject();
|
||
credentialsJsonParam.put("parentId", "0010k00001eVKlZAAW");
|
||
credentialsJsonParam.put("Name", fileName);
|
||
MultipartEntityBuilder builder = MultipartEntityBuilder.create();
|
||
builder.addTextBody("data", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON);
|
||
builder.addBinaryBody("Body", new FileInputStream(file), ContentType.APPLICATION_OCTET_STREAM, fileName);
|
||
HttpEntity entity = builder.build();
|
||
httpPost.setEntity(entity);
|
||
|
||
CloseableHttpClient httpClient = HttpClients.createDefault();
|
||
response = httpClient.execute(httpPost);
|
||
try {
|
||
if (response != null && response.getStatusLine() != null && response.getStatusLine().getStatusCode() < 400) {
|
||
HttpEntity he = response.getEntity();
|
||
if (he != null) {
|
||
respContent = EntityUtils.toString(he, "UTF-8");
|
||
}
|
||
System.out.println("------------------------------"+JSONObject.toJSONString(respContent));
|
||
} else {
|
||
throw new RuntimeException();
|
||
}
|
||
} catch (Exception e) {
|
||
// log.error("网络访问异常,请求url地址={},响应体={},error={}", url, response, e);
|
||
throw new RuntimeException();
|
||
} finally {
|
||
if (response != null) {
|
||
response.close();
|
||
}
|
||
}
|
||
|
||
System.out.println(JSONObject.parseObject(respContent).get("id"));
|
||
}
|
||
|
||
@Test
|
||
public void madeBatch() throws Exception {
|
||
List<DataObject> list = dataObjectService.list();
|
||
Date now = new Date();
|
||
for (DataObject dataObject : list) {
|
||
String apiName = dataObject.getName();
|
||
String label = dataObject.getLabel();
|
||
// 生成batch
|
||
Date startDate = DataUtil.DEFAULT_BEGIN_DATE;
|
||
// 取当天零点 作为最大时间
|
||
Date endCreateDate = DateUtils.parseDate(DateFormatUtils.format(now, "yyyy-MM-dd"), "yyyy-MM-dd");
|
||
Date lastDay = null;
|
||
QueryWrapper<DataBatch> delQw = new QueryWrapper<>();
|
||
delQw.eq("name", apiName);
|
||
dataBatchService.remove(delQw);
|
||
if (true && Const.BATCH_FILTERS.stream().noneMatch(t -> apiName.indexOf(t) > 0)) {
|
||
// 匹配的不生成
|
||
DataBatch one = new DataBatch();
|
||
one.setFirstDbNum(0);
|
||
one.setFirstSfNum(0);
|
||
one.setDbNum(0);
|
||
one.setSfNum(0);
|
||
one.setName(apiName);
|
||
one.setLabel(label);
|
||
if (true) {
|
||
do {
|
||
lastDay = DataUtil.getWeekLastDay(endCreateDate, startDate);
|
||
DataBatch dataBatch = one.clone();
|
||
dataBatch.setSyncStartDate(startDate);
|
||
dataBatch.setSyncEndDate(lastDay);
|
||
dataBatchService.save(dataBatch);
|
||
startDate = lastDay;
|
||
} while (lastDay.compareTo(endCreateDate) < 0);
|
||
} else {
|
||
// 没创建时间 开始结束时间为空
|
||
dataBatchService.save(one);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|