From 58204692a6e240c3aa6642d2f75ad6e12cb83f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=9C=E5=88=9A?= <53732908@qq.com> Date: Wed, 12 Feb 2025 14:03:24 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=AD=97=E6=AE=B5=E5=92=8C?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E6=9C=AC=E5=9C=B0=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dao/datax/datamanage/FieldMapper.java | 2 + .../dao/datax/datamanage/FieldMapper.xml | 6 + .../aimodel/MachineLearningModelsMapper.java | 3 + .../aimodel/MachineLearningModelsMapper.xml | 8 + .../model/strategyx/aimodel/ModelDTO.java | 56 ++++++ .../service/cache/LocalCacheService.java | 182 ++++++++++++++++++ .../ddp/common/service/common/CommUtil.java | 18 ++ .../src/main/resources/application.properties | 2 + .../aimodel/controller/ModelsController.java | 27 ++- 9 files changed, 302 insertions(+), 2 deletions(-) create mode 100644 ddp/ddp-common/ddp-model/src/main/java/com/fibo/ddp/common/model/strategyx/aimodel/ModelDTO.java create mode 100644 ddp/ddp-common/ddp-service/src/main/java/com/fibo/ddp/common/service/cache/LocalCacheService.java create mode 100644 ddp/ddp-common/ddp-service/src/main/java/com/fibo/ddp/common/service/common/CommUtil.java diff --git a/ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/datax/datamanage/FieldMapper.java b/ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/datax/datamanage/FieldMapper.java index e304f8a..d9de176 100644 --- a/ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/datax/datamanage/FieldMapper.java +++ b/ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/datax/datamanage/FieldMapper.java @@ -167,4 +167,6 @@ public interface FieldMapper extends BaseMapper { public Field findByFieldCnbyorganId(Map paramMap); List selectFieldListByEns(@Param("ens")List ens); + + List getAllFieldList(); } diff --git a/ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/datax/datamanage/FieldMapper.xml b/ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/datax/datamanage/FieldMapper.xml index 0b0501e..39b93f5 100644 --- a/ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/datax/datamanage/FieldMapper.xml +++ b/ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/datax/datamanage/FieldMapper.xml @@ -367,6 +367,12 @@ + + + + + \ No newline at end of file diff --git a/ddp/ddp-common/ddp-model/src/main/java/com/fibo/ddp/common/model/strategyx/aimodel/ModelDTO.java b/ddp/ddp-common/ddp-model/src/main/java/com/fibo/ddp/common/model/strategyx/aimodel/ModelDTO.java new file mode 100644 index 0000000..b8d9946 --- /dev/null +++ b/ddp/ddp-common/ddp-model/src/main/java/com/fibo/ddp/common/model/strategyx/aimodel/ModelDTO.java @@ -0,0 +1,56 @@ +package com.fibo.ddp.common.model.strategyx.aimodel; + +import org.jpmml.evaluator.Evaluator; + +import java.time.LocalDateTime; + +public class ModelDTO { + + public String createTime = LocalDateTime.now().toString(); + private MachineLearningModels model; + private String errorMsg; + + private String text; + + private Evaluator evaluator; + + public String getCreateTime() { + return createTime; + } + + public void setCreateTime(String createTime) { + this.createTime = createTime; + } + + public MachineLearningModels getModel() { + return model; + } + + public void setModel(MachineLearningModels model) { + this.model = model; + } + + public String getErrorMsg() { + return errorMsg; + } + + public void setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + } + + public Evaluator getEvaluator() { + return evaluator; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public void setEvaluator(Evaluator evaluator) { + this.evaluator = evaluator; + } +} diff --git a/ddp/ddp-common/ddp-service/src/main/java/com/fibo/ddp/common/service/cache/LocalCacheService.java b/ddp/ddp-common/ddp-service/src/main/java/com/fibo/ddp/common/service/cache/LocalCacheService.java new file mode 100644 index 0000000..207ae2e --- /dev/null +++ b/ddp/ddp-common/ddp-service/src/main/java/com/fibo/ddp/common/service/cache/LocalCacheService.java @@ -0,0 +1,182 @@ +package com.fibo.ddp.common.service.cache; + + +import com.fibo.ddp.common.dao.datax.datamanage.FieldMapper; +import com.fibo.ddp.common.dao.strategyx.aimodel.MachineLearningModelsMapper; +import com.fibo.ddp.common.model.datax.datamanage.Field; +import com.fibo.ddp.common.model.strategyx.aimodel.MachineLearningModels; +import com.fibo.ddp.common.model.strategyx.aimodel.ModelDTO; +import com.fibo.ddp.common.service.common.CommUtil; +import com.fibo.ddp.common.service.strategyx.aimodel.PMMLExecutor.PMMLExecutor; +import org.jpmml.evaluator.Evaluator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +@Service +public class LocalCacheService { + + private static final Logger logger = LoggerFactory.getLogger(LocalCacheService.class); + + + // + private static final long ONE_SECOND_MS = 1000L; + + // 10s + private static final long UPDATE_SLEEP_TIME_MS = 10 * ONE_SECOND_MS; + + + + // + + private final AtomicLong runCount = new AtomicLong(0); + // key is id + private volatile Map fieldsMap = new ConcurrentHashMap<>(); + private volatile Map modelMap = new ConcurrentHashMap<>(); + + private final Map infoMap = new ConcurrentHashMap<>(); + + // + + @Resource + FieldMapper fieldMapper; + + @Autowired + MachineLearningModelsMapper machineLearningModelsMapper; + + @Resource + PMMLExecutor pmmlExecutor; + + + + + @PostConstruct + public void init() { + + + logger.info("LocalCacheService_init_start"); + updateAllCache(true); + // + CommUtil.doSleep(UPDATE_SLEEP_TIME_MS); + + Thread t = new Thread(); + t.start(); + logger.info("LocalCacheService_init_done"); + + + } + + public Map getInfo(){ + Map info = new HashMap<>(); + info.put("runCount",runCount.longValue()); + info.put("info",infoMap); + + Map fieldsMapTmp = fieldsMap; + Map modelMapTmp = modelMap; + + if(fieldsMapTmp!=null){ + info.put("fieldsNum",fieldsMapTmp.size()); + } + + if(modelMapTmp!=null){ + info.put("modelNum",modelMapTmp.size()); + } + + return info; + } + + public Map getFieldsMap(){ + return fieldsMap; + } + + public Map getModelMap(){ + return modelMap; + } + + public void updateModelCache()throws Exception{ + + Map modelMapTmp = new ConcurrentHashMap<>(); + List models = machineLearningModelsMapper.getAllModelList(); + + if(models!=null){ + for(MachineLearningModels item:models){ + Integer id = item.getId(); + if(id==null){ + continue; + } + String filePath = item.getFilePath(); + // ModelDTO modelDTOOld = modelMap.get(id+""); + ModelDTO modelDTO = new ModelDTO(); + modelDTO.setModel(item); + try { + Evaluator evaluator = pmmlExecutor.loadPmml(filePath); + modelDTO.setEvaluator(evaluator); + if (evaluator == null) { + modelDTO.setErrorMsg("evaluator_null"); + } + }catch(Throwable e){ + modelDTO.setErrorMsg("loadPmml_error,"+e); + } + modelMapTmp.put(id+"",modelDTO); + } + } + + modelMap = modelMapTmp; + + } + + public void updateAllCache(boolean throwsError){ + boolean error = false; + try{ + updateFieldsCache(); + }catch(Throwable e){ + error = true; + logger.error("updateFieldsCache_error",e); + } + + try{ + updateModelCache(); + }catch(Throwable e){ + error = true; + logger.error("updateModelCache_error",e); + } + if(throwsError && error){ + throw new RuntimeException("updateAllCacheError"); + } + } + + public void updateFieldsCache(){ + Map fieldsMapTmp = new ConcurrentHashMap<>(); + List fields = fieldMapper.getAllFieldList(); + if(fields!=null) { + for(Field item:fields){ + Long id = item.getId(); + if(id!=null){ + fieldsMapTmp.put(id+"",item); + } + } + } + fieldsMap = fieldsMapTmp; + } + + + + private class LocalCacheUpdateThread extends Thread{ + public void run(){ + runCount.getAndIncrement(); + updateAllCache(false); + } + } + + + +} diff --git a/ddp/ddp-common/ddp-service/src/main/java/com/fibo/ddp/common/service/common/CommUtil.java b/ddp/ddp-common/ddp-service/src/main/java/com/fibo/ddp/common/service/common/CommUtil.java new file mode 100644 index 0000000..7048226 --- /dev/null +++ b/ddp/ddp-common/ddp-service/src/main/java/com/fibo/ddp/common/service/common/CommUtil.java @@ -0,0 +1,18 @@ +package com.fibo.ddp.common.service.common; + +public class CommUtil { + + + public static void doSleep(long time){ + if(time<=0){ + return; + } + try{ + Thread.sleep(time); + }catch(Throwable e){ + // do nothing + } + } + + +} diff --git a/ddp/ddp-runner-api/src/main/resources/application.properties b/ddp/ddp-runner-api/src/main/resources/application.properties index d905b31..df187ba 100644 --- a/ddp/ddp-runner-api/src/main/resources/application.properties +++ b/ddp/ddp-runner-api/src/main/resources/application.properties @@ -60,5 +60,7 @@ canal.port=11111 # monitor monitor.data.storage.type=mysql +model.uploadDir=/tmp/model/upload/ + diff --git a/ddp/ddp-strategyx/ai-model/src/main/java/com/fibo/ddp/strategyx/aimodel/controller/ModelsController.java b/ddp/ddp-strategyx/ai-model/src/main/java/com/fibo/ddp/strategyx/aimodel/controller/ModelsController.java index 4a4875c..f5c7cbe 100644 --- a/ddp/ddp-strategyx/ai-model/src/main/java/com/fibo/ddp/strategyx/aimodel/controller/ModelsController.java +++ b/ddp/ddp-strategyx/ai-model/src/main/java/com/fibo/ddp/strategyx/aimodel/controller/ModelsController.java @@ -21,6 +21,7 @@ import org.jpmml.evaluator.Evaluator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartHttpServletRequest; @@ -50,6 +51,13 @@ public class ModelsController { @Autowired private MachineLearningModelsMapper machineLearningModelsMapper; + @Value("${model.uploadDir}") + private String modelUploadDir; + + private static final String SLASH = "/"; + + + /** * 获取模型列表信息 * @return @@ -110,13 +118,16 @@ public class ModelsController { while (iter.hasNext()) { MultipartFile file = multiRequest.getFile(iter.next().toString()); if (file != null) { - String uploadDir = request.getSession().getServletContext().getRealPath("/") + "upload/models/fieldUpload/"; + // String uploadDir = request.getSession().getServletContext().getRealPath("/") + "upload/models/fieldUpload/"; + final String uploadDir = modelUploadDir; + logger.info("modelUploadDir="+modelUploadDir); if (!new File(uploadDir).exists()) { File dir = new File(uploadDir); dir.mkdirs(); } fileName = System.currentTimeMillis() + "_" + file.getOriginalFilename(); - String path = uploadDir + fileName; + final String path = buildPath(uploadDir,fileName); + logger.info("modelFilePath"+path); //上传 file.transferTo(new File(path)); accessUrl = path; @@ -138,6 +149,18 @@ public class ModelsController { } } + @RequestMapping(value = "getModelUploadDir") + public String getModelUploadDir(){ + return modelUploadDir; + } + + private String buildPath(String dir,String fileName){ + if(dir.endsWith(SLASH)){ + return dir + fileName; + } + return dir + SLASH + fileName; + } + /** * 添加模型 * @param models