增加字段和模型本地缓存

ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/datax/datamanage/FieldMapper.java

@@ -167,4 +167,6 @@ public interface FieldMapper extends BaseMapper<Field> {
     public Field findByFieldCnbyorganId(Map<String, Object> paramMap);
 
     List<Field> selectFieldListByEns(@Param("ens")List<String> ens);
+
+    List<Field> getAllFieldList();
 }

ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/datax/datamanage/FieldMapper.xml

@@ -367,6 +367,12 @@
         <!--                    )-->
     </update>
 
+    <select id="getAllFieldList" resultType="com.fibo.ddp.common.model.datax.datamanage.Field">
+        select
+        <include refid="Base_Column"></include>
+        from t_field
+    </select>
+
     <select id="getFieldList" parameterType="map" resultType="com.fibo.ddp.common.model.datax.datamanage.Field">
         select
         t.id,

ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/strategyx/aimodel/MachineLearningModelsMapper.java

@@ -16,4 +16,7 @@ public interface MachineLearningModelsMapper extends BaseMapper<MachineLearningM
      * @return
      */
     List<MachineLearningModels> getModelsListByOrganId(@Param("organId") Integer organId, @Param("searchString") String searchString);
+    List<MachineLearningModels> getAllModelList();
+
+
 }

ddp/ddp-common/ddp-dao/src/main/java/com/fibo/ddp/common/dao/strategyx/aimodel/MachineLearningModelsMapper.xml

@@ -15,4 +15,12 @@
        and (model_name like CONCAT('%',TRIM('${searchString}'),'%' ) or description like CONCAT('%',TRIM('${searchString}'),'%' ))
     </if>
   </select>
+
+  <select id="getAllModelList" resultType="com.fibo.ddp.common.model.strategyx.aimodel.MachineLearningModels">
+    select
+    <include refid="Base_Column_List" />
+    from t_machine_learning_models
+    where status = 1
+  </select>
+
 </mapper>

ddp/ddp-common/ddp-model/src/main/java/com/fibo/ddp/common/model/strategyx/aimodel/ModelDTO.java

@@ -0,0 +1,56 @@
+package com.fibo.ddp.common.model.strategyx.aimodel;
+
+import org.jpmml.evaluator.Evaluator;
+
+import java.time.LocalDateTime;
+
+public class ModelDTO {
+
+    public String createTime = LocalDateTime.now().toString();
+    private MachineLearningModels model;
+    private String errorMsg;
+
+    private String text;
+
+    private Evaluator evaluator;
+
+    public String getCreateTime() {
+        return createTime;
+    }
+
+    public void setCreateTime(String createTime) {
+        this.createTime = createTime;
+    }
+
+    public MachineLearningModels getModel() {
+        return model;
+    }
+
+    public void setModel(MachineLearningModels model) {
+        this.model = model;
+    }
+
+    public String getErrorMsg() {
+        return errorMsg;
+    }
+
+    public void setErrorMsg(String errorMsg) {
+        this.errorMsg = errorMsg;
+    }
+
+    public Evaluator getEvaluator() {
+        return evaluator;
+    }
+
+    public String getText() {
+        return text;
+    }
+
+    public void setText(String text) {
+        this.text = text;
+    }
+
+    public void setEvaluator(Evaluator evaluator) {
+        this.evaluator = evaluator;
+    }
+}

ddp/ddp-common/ddp-service/src/main/java/com/fibo/ddp/common/service/cache/LocalCacheService.java

@@ -0,0 +1,182 @@
+package com.fibo.ddp.common.service.cache;
+
+
+import com.fibo.ddp.common.dao.datax.datamanage.FieldMapper;
+import com.fibo.ddp.common.dao.strategyx.aimodel.MachineLearningModelsMapper;
+import com.fibo.ddp.common.model.datax.datamanage.Field;
+import com.fibo.ddp.common.model.strategyx.aimodel.MachineLearningModels;
+import com.fibo.ddp.common.model.strategyx.aimodel.ModelDTO;
+import com.fibo.ddp.common.service.common.CommUtil;
+import com.fibo.ddp.common.service.strategyx.aimodel.PMMLExecutor.PMMLExecutor;
+import org.jpmml.evaluator.Evaluator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.Resource;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Service
+public class LocalCacheService {
+
+    private static final Logger logger = LoggerFactory.getLogger(LocalCacheService.class);
+
+
+    //
+    private static final long ONE_SECOND_MS = 1000L;
+
+    // 10s
+    private static final long UPDATE_SLEEP_TIME_MS = 10 * ONE_SECOND_MS;
+
+
+
+    //
+
+    private final AtomicLong runCount = new AtomicLong(0);
+    // key is id
+    private  volatile Map<String, Field> fieldsMap = new ConcurrentHashMap<>();
+    private  volatile Map<String, ModelDTO> modelMap = new ConcurrentHashMap<>();
+
+    private  final Map<String, String> infoMap = new ConcurrentHashMap<>();
+
+    //
+
+    @Resource
+    FieldMapper fieldMapper;
+
+    @Autowired
+    MachineLearningModelsMapper machineLearningModelsMapper;
+
+    @Resource
+    PMMLExecutor pmmlExecutor;
+
+
+
+
+    @PostConstruct
+    public void init() {
+
+
+        logger.info("LocalCacheService_init_start");
+        updateAllCache(true);
+        //
+        CommUtil.doSleep(UPDATE_SLEEP_TIME_MS);
+
+        Thread t = new Thread();
+        t.start();
+        logger.info("LocalCacheService_init_done");
+
+
+    }
+
+    public Map<String,Object> getInfo(){
+        Map<String,Object> info = new HashMap<>();
+        info.put("runCount",runCount.longValue());
+        info.put("info",infoMap);
+
+        Map fieldsMapTmp = fieldsMap;
+        Map modelMapTmp = modelMap;
+
+        if(fieldsMapTmp!=null){
+            info.put("fieldsNum",fieldsMapTmp.size());
+        }
+
+        if(modelMapTmp!=null){
+            info.put("modelNum",modelMapTmp.size());
+        }
+
+        return info;
+    }
+
+    public  Map<String, Field> getFieldsMap(){
+        return fieldsMap;
+    }
+
+    public  Map<String, ModelDTO> getModelMap(){
+        return modelMap;
+    }
+
+    public void updateModelCache()throws Exception{
+
+        Map<String, ModelDTO> modelMapTmp = new ConcurrentHashMap<>();
+        List<MachineLearningModels> models =  machineLearningModelsMapper.getAllModelList();
+
+        if(models!=null){
+            for(MachineLearningModels item:models){
+                Integer id = item.getId();
+                if(id==null){
+                    continue;
+                }
+                String filePath = item.getFilePath();
+                // ModelDTO modelDTOOld = modelMap.get(id+"");
+                ModelDTO modelDTO = new ModelDTO();
+                modelDTO.setModel(item);
+               try {
+                   Evaluator evaluator = pmmlExecutor.loadPmml(filePath);
+                   modelDTO.setEvaluator(evaluator);
+                   if (evaluator == null) {
+                       modelDTO.setErrorMsg("evaluator_null");
+                   }
+               }catch(Throwable e){
+                   modelDTO.setErrorMsg("loadPmml_error,"+e);
+               }
+                modelMapTmp.put(id+"",modelDTO);
+            }
+        }
+
+        modelMap = modelMapTmp;
+
+    }
+
+    public void updateAllCache(boolean throwsError){
+        boolean error = false;
+        try{
+            updateFieldsCache();
+        }catch(Throwable e){
+            error = true;
+            logger.error("updateFieldsCache_error",e);
+        }
+
+        try{
+            updateModelCache();
+        }catch(Throwable e){
+            error = true;
+            logger.error("updateModelCache_error",e);
+        }
+        if(throwsError && error){
+            throw new RuntimeException("updateAllCacheError");
+        }
+    }
+
+    public void updateFieldsCache(){
+        Map<String, Field> fieldsMapTmp = new ConcurrentHashMap<>();
+        List<Field> fields =  fieldMapper.getAllFieldList();
+        if(fields!=null) {
+            for(Field item:fields){
+                Long id = item.getId();
+                if(id!=null){
+                    fieldsMapTmp.put(id+"",item);
+                }
+            }
+        }
+        fieldsMap = fieldsMapTmp;
+    }
+
+
+
+    private class LocalCacheUpdateThread extends Thread{
+        public void run(){
+            runCount.getAndIncrement();
+            updateAllCache(false);
+        }
+    }
+
+
+
+}

ddp/ddp-common/ddp-service/src/main/java/com/fibo/ddp/common/service/common/CommUtil.java

@@ -0,0 +1,18 @@
+package com.fibo.ddp.common.service.common;
+
+public class CommUtil {
+
+
+    public static void doSleep(long time){
+        if(time<=0){
+            return;
+        }
+        try{
+            Thread.sleep(time);
+        }catch(Throwable e){
+            // do nothing
+        }
+    }
+
+
+}

ddp/ddp-runner-api/src/main/resources/application.properties

@@ -60,5 +60,7 @@ canal.port=11111
 # monitor
 monitor.data.storage.type=mysql 
 
+model.uploadDir=/tmp/model/upload/
+
 
 

ddp/ddp-strategyx/ai-model/src/main/java/com/fibo/ddp/strategyx/aimodel/controller/ModelsController.java

@@ -21,6 +21,7 @@ import org.jpmml.evaluator.Evaluator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 import org.springframework.web.multipart.MultipartHttpServletRequest;
@@ -50,6 +51,13 @@ public class ModelsController {
     @Autowired
     private MachineLearningModelsMapper machineLearningModelsMapper;
 
+    @Value("${model.uploadDir}")
+    private String modelUploadDir;
+
+    private static final String SLASH = "/";
+
+
+
     /**
      * 获取模型列表信息
      * @return
@@ -110,13 +118,16 @@ public class ModelsController {
             while (iter.hasNext()) {
                 MultipartFile file = multiRequest.getFile(iter.next().toString());
                 if (file != null) {
-                    String uploadDir = request.getSession().getServletContext().getRealPath("/") + "upload/models/fieldUpload/";
+                    // String uploadDir = request.getSession().getServletContext().getRealPath("/") + "upload/models/fieldUpload/";
+                    final String uploadDir = modelUploadDir;
+                    logger.info("modelUploadDir="+modelUploadDir);
                     if (!new File(uploadDir).exists()) {
                         File dir = new File(uploadDir);
                         dir.mkdirs();
                     }
                     fileName = System.currentTimeMillis() + "_" + file.getOriginalFilename();
-                    String path = uploadDir + fileName;
+                    final String path = buildPath(uploadDir,fileName);
+                    logger.info("modelFilePath"+path);
                     //上传
                     file.transferTo(new File(path));
                     accessUrl = path;
@@ -138,6 +149,18 @@ public class ModelsController {
         }
     }
 
+    @RequestMapping(value = "getModelUploadDir")
+    public String getModelUploadDir(){
+        return modelUploadDir;
+    }
+
+    private String buildPath(String dir,String fileName){
+        if(dir.endsWith(SLASH)){
+            return dir + fileName;
+        }
+        return dir + SLASH  + fileName;
+    }
+
     /**
      * 添加模型
      * @param models