liutongsz/fibo-rule
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加字段和模型本地缓存

Browse Source
This commit is contained in:
杜刚
2025-02-12 14:03:24 +08:00
parent 554286c88c
commit 58204692a6
9 changed files with 302 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/datax/datamanage/FieldMapper.java
View File

@@ -167,4 +167,6 @@ public interface FieldMapper extends BaseMapper<Field> {
public Field findByFieldCnbyorganId(Map<String, Object> paramMap);
List<Field> selectFieldListByEns(@Param("ens")List<String> ens);
List<Field> getAllFieldList();
}

6
ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/datax/datamanage/FieldMapper.xml
View File

@@ -367,6 +367,12 @@
<!-- )-->
</update>
<select id="getAllFieldList" resultType="com.fibo.ddp.common.model.datax.datamanage.Field">
select
<include refid="Base_Column"></include>
from t_field
</select>
<select id="getFieldList" parameterType="map" resultType="com.fibo.ddp.common.model.datax.datamanage.Field">
select
t.id,

3
ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/strategyx/aimodel/MachineLearningModelsMapper.java
View File

@@ -16,4 +16,7 @@ public interface MachineLearningModelsMapper extends BaseMapper<MachineLearningM
* @return
*/
List<MachineLearningModels> getModelsListByOrganId(@Param("organId") Integer organId, @Param("searchString") String searchString);
List<MachineLearningModels> getAllModelList();
}

8
ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/strategyx/aimodel/MachineLearningModelsMapper.xml
View File

@@ -15,4 +15,12 @@
and (model_name like CONCAT('%',TRIM('${searchString}'),'%' ) or description like CONCAT('%',TRIM('${searchString}'),'%' ))
</if>
</select>
<select id="getAllModelList" resultType="com.fibo.ddp.common.model.strategyx.aimodel.MachineLearningModels">
select
<include refid="Base_Column_List" />
from t_machine_learning_models
where status = 1
</select>
</mapper>

56
ddp/ddp-common/ddp-model/src/main/java/com/fibo/ddp/common/model/strategyx/aimodel/ModelDTO.java Normal file
View File

@@ -0,0 +1,56 @@
package com.fibo.ddp.common.model.strategyx.aimodel;
import org.jpmml.evaluator.Evaluator;
import java.time.LocalDateTime;
public class ModelDTO {
public String createTime = LocalDateTime.now().toString();
private MachineLearningModels model;
private String errorMsg;
private String text;
private Evaluator evaluator;
public String getCreateTime() {
return createTime;
}
public void setCreateTime(String createTime) {
this.createTime = createTime;
}
public MachineLearningModels getModel() {
return model;
}
public void setModel(MachineLearningModels model) {
this.model = model;
}
public String getErrorMsg() {
return errorMsg;
}
public void setErrorMsg(String errorMsg) {
this.errorMsg = errorMsg;
}
public Evaluator getEvaluator() {
return evaluator;
}
public String getText() {
return text;
}
public void setText(String text) {
this.text = text;
}
public void setEvaluator(Evaluator evaluator) {
this.evaluator = evaluator;
}
}

182
ddp/ddp-common/ddp-service/src/main/java/com/fibo/ddp/common/service/cache/LocalCacheService.java vendored Normal file
View File

@@ -0,0 +1,182 @@
package com.fibo.ddp.common.service.cache;
import com.fibo.ddp.common.dao.datax.datamanage.FieldMapper;
import com.fibo.ddp.common.dao.strategyx.aimodel.MachineLearningModelsMapper;
import com.fibo.ddp.common.model.datax.datamanage.Field;
import com.fibo.ddp.common.model.strategyx.aimodel.MachineLearningModels;
import com.fibo.ddp.common.model.strategyx.aimodel.ModelDTO;
import com.fibo.ddp.common.service.common.CommUtil;
import com.fibo.ddp.common.service.strategyx.aimodel.PMMLExecutor.PMMLExecutor;
import org.jpmml.evaluator.Evaluator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
@Service
public class LocalCacheService {
private static final Logger logger = LoggerFactory.getLogger(LocalCacheService.class);
//
private static final long ONE_SECOND_MS = 1000L;
// 10s
private static final long UPDATE_SLEEP_TIME_MS = 10 * ONE_SECOND_MS;
//
private final AtomicLong runCount = new AtomicLong(0);
// key is id
private volatile Map<String, Field> fieldsMap = new ConcurrentHashMap<>();
private volatile Map<String, ModelDTO> modelMap = new ConcurrentHashMap<>();
private final Map<String, String> infoMap = new ConcurrentHashMap<>();
//
@Resource
FieldMapper fieldMapper;
@Autowired
MachineLearningModelsMapper machineLearningModelsMapper;
@Resource
PMMLExecutor pmmlExecutor;
@PostConstruct
public void init() {
logger.info("LocalCacheService_init_start");
updateAllCache(true);
//
CommUtil.doSleep(UPDATE_SLEEP_TIME_MS);
Thread t = new Thread();
t.start();
logger.info("LocalCacheService_init_done");
}
public Map<String,Object> getInfo(){
Map<String,Object> info = new HashMap<>();
info.put("runCount",runCount.longValue());
info.put("info",infoMap);
Map fieldsMapTmp = fieldsMap;
Map modelMapTmp = modelMap;
if(fieldsMapTmp!=null){
info.put("fieldsNum",fieldsMapTmp.size());
}
if(modelMapTmp!=null){
info.put("modelNum",modelMapTmp.size());
}
return info;
}
public Map<String, Field> getFieldsMap(){
return fieldsMap;
}
public Map<String, ModelDTO> getModelMap(){
return modelMap;
}
public void updateModelCache()throws Exception{
Map<String, ModelDTO> modelMapTmp = new ConcurrentHashMap<>();
List<MachineLearningModels> models = machineLearningModelsMapper.getAllModelList();
if(models!=null){
for(MachineLearningModels item:models){
Integer id = item.getId();
if(id==null){
continue;
}
String filePath = item.getFilePath();
// ModelDTO modelDTOOld = modelMap.get(id+"");
ModelDTO modelDTO = new ModelDTO();
modelDTO.setModel(item);
try {
Evaluator evaluator = pmmlExecutor.loadPmml(filePath);
modelDTO.setEvaluator(evaluator);
if (evaluator == null) {
modelDTO.setErrorMsg("evaluator_null");
}
}catch(Throwable e){
modelDTO.setErrorMsg("loadPmml_error,"+e);
}
modelMapTmp.put(id+"",modelDTO);
}
}
modelMap = modelMapTmp;
}
public void updateAllCache(boolean throwsError){
boolean error = false;
try{
updateFieldsCache();
}catch(Throwable e){
error = true;
logger.error("updateFieldsCache_error",e);
}
try{
updateModelCache();
}catch(Throwable e){
error = true;
logger.error("updateModelCache_error",e);
}
if(throwsError && error){
throw new RuntimeException("updateAllCacheError");
}
}
public void updateFieldsCache(){
Map<String, Field> fieldsMapTmp = new ConcurrentHashMap<>();
List<Field> fields = fieldMapper.getAllFieldList();
if(fields!=null) {
for(Field item:fields){
Long id = item.getId();
if(id!=null){
fieldsMapTmp.put(id+"",item);
}
}
}
fieldsMap = fieldsMapTmp;
}
private class LocalCacheUpdateThread extends Thread{
public void run(){
runCount.getAndIncrement();
updateAllCache(false);
}
}
}

18
ddp/ddp-common/ddp-service/src/main/java/com/fibo/ddp/common/service/common/CommUtil.java Normal file
View File

@@ -0,0 +1,18 @@
package com.fibo.ddp.common.service.common;
public class CommUtil {
public static void doSleep(long time){
if(time<=0){
return;
}
try{
Thread.sleep(time);
}catch(Throwable e){
// do nothing
}
}
}

2
ddp/ddp-runner-api/src/main/resources/application.properties
View File

@@ -60,5 +60,7 @@ canal.port=11111
# monitor
monitor.data.storage.type=mysql
model.uploadDir=/tmp/model/upload/

27
ddp/ddp-strategyx/ai-model/src/main/java/com/fibo/ddp/strategyx/aimodel/controller/ModelsController.java
View File

@@ -21,6 +21,7 @@ import org.jpmml.evaluator.Evaluator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
@@ -50,6 +51,13 @@ public class ModelsController {
@Autowired
private MachineLearningModelsMapper machineLearningModelsMapper;
@Value("${model.uploadDir}")
private String modelUploadDir;
private static final String SLASH = "/";
/**
* 获取模型列表信息
* @return
@@ -110,13 +118,16 @@ public class ModelsController {
while (iter.hasNext()) {
MultipartFile file = multiRequest.getFile(iter.next().toString());
if (file != null) {
String uploadDir = request.getSession().getServletContext().getRealPath("/") + "upload/models/fieldUpload/";
// String uploadDir = request.getSession().getServletContext().getRealPath("/") + "upload/models/fieldUpload/";
final String uploadDir = modelUploadDir;
logger.info("modelUploadDir="+modelUploadDir);
if (!new File(uploadDir).exists()) {
File dir = new File(uploadDir);
dir.mkdirs();
}
fileName = System.currentTimeMillis() + "_" + file.getOriginalFilename();
String path = uploadDir + fileName;
final String path = buildPath(uploadDir,fileName);
logger.info("modelFilePath"+path);
//上传
file.transferTo(new File(path));
accessUrl = path;
@@ -138,6 +149,18 @@ public class ModelsController {
}
}
@RequestMapping(value = "getModelUploadDir")
public String getModelUploadDir(){
return modelUploadDir;
}
private String buildPath(String dir,String fileName){
if(dir.endsWith(SLASH)){
return dir + fileName;
}
return dir + SLASH + fileName;
}
/**
* 添加模型
* @param models