Refactorings

Documentation
Example code to show the usage of the webservice callback

entity/DuplicateScanner_entity/entityfields/testactiongroup/children/testduplicatescanner/onActionProcess.js

@@ -186,13 +186,15 @@ import("JditoFilter_lib");
 var filterName = "PersonDuplicates";
 var targetEntity = "Person_entity";
 
-
-let duplicateFieldsConfig = DuplicateScannerUtils.LoadDuplicateIndexFieldsConfiguration(filterName, targetEntity);
+let duplicateFieldsConfig = DuplicateScannerUtils.LoadIndexFieldsConfiguration(filterName, targetEntity);
+let resultFields = DuplicateScannerUtils.LoadResultFields(filterName, targetEntity);
 
 logging.log("duplicateFieldsConfig -> " + duplicateFieldsConfig);
+logging.log("resultFields -> " + resultFields);
 
 let querySelectFields = DuplicateScannerUtils.BuildSqlSelectFieldsFromFieldConfig(duplicateFieldsConfig);
 logging.log("querySelectFields -> " + querySelectFields);
+
 let queryPersonFieldData = "select " + querySelectFields + " from CONTACT"
                             + " join PERSON on PERSONID = PERSON_ID"
                             + " left join ADDRESS on ADDRESS.CONTACT_ID = CONTACT.CONTACTID";
@@ -200,17 +202,28 @@ let queryPersonFieldData = "select " + querySelectFields + " from CONTACT"
 logging.log("Löschen von PERSON Dubletten -> ");
 DuplicateScannerUtils.DeleteDuplicateClustersByTargetEntity(targetEntity);
 
-let formatToJsonAndCallWs = function(pPossibleDuplicatesRay) {
-    logging.log("pPossibleDuplicatesRay  -> " + pPossibleDuplicatesRay);
+let formatToJsonAndCallWsCallback = function(pPossibleDuplicatesRay) {
     logging.log(" bin in functioin -> " + pPossibleDuplicatesRay.length);
-    if(pPossibleDuplicatesRay.length > 0)
-        logging.log("DemoIdFeld -> " + pPossibleDuplicatesRay[0][indexsearch.FIELD_ID]);
+    
+    let indexResultFields = DuplicateScannerUtils.TranslateEntityToIndexFields(targetEntity, resultFields)
+    
+    for (let i = 0; i < pPossibleDuplicatesRay.length; i++) 
+    {
+        for (let b = 0; b < resultFields.length; b++) 
+        {
+            let entityFieldName = resultFields[b];
+            let indexFieldName = indexResultFields[entityFieldName];
+            logging.log("entityFieldName -> " + entityFieldName);
+            logging.log("indexFieldName -> " + indexFieldName);
+            logging.log("Entity Field -> "+ pPossibleDuplicatesRay[i][indexFieldName]);
+        }
+    }
     logging.log("wieder draußen -> ");
     return pPossibleDuplicatesRay;
 };
 
 logging.log("Neu berechnen von PERSON Dubletten -> ");
 DuplicateScannerUtils.RebuildDuplicatesCache(filterName, targetEntity, queryPersonFieldData,
-duplicateFieldsConfig, formatToJsonAndCallWs);
+duplicateFieldsConfig, resultFields, formatToJsonAndCallWsCallback);
 
 DuplicateScannerUtils.RefreshUnrelatedDuplicateRelations(targetEntity);
\ No newline at end of file

process/DuplicateScanner_lib/process.js

@@ -204,9 +204,15 @@ DuplicateScannerUtils.GetCachedDuplicatesForId = function(pClusterRecordId)
  * The old clusters have to be deleted manually beforehand using "DeleteDuplicateClustersByTargetEntity".
  * If there have already been ignored relations between duplicate records, it's advised to call "RefreshUnrelatedDuplicateRelations" after the recreation of the duplicates cache.
  * 
+ * Attention!
+ * If it is configured to use the external webservice callback the values have to be in the same format as they are in the parameter of the callback.
+ * 
  * @param {String} pFilterName Name of the filter to use
  * @param {String} pTargetEntity The target entity which has been assigned to the filters configuration
  * @param {String} pQueryTargetRecords Query which holds the values that are being used as configured in the filter.
+ * @param {String} pDuplicateFieldsConfig 
+ * @param {String} pResultFields todo
+ * @param {String} pFormatValuesConsumeWebserviceCallback 
  * Important: The first element has to be the id field!
  * @param {String[]} pDuplicateFieldsConfig The configuration of the fields and their usage. @see DuplicateScannerUtils.LoadDuplicateIndexFieldsConfiguration
  *
@@ -229,7 +235,7 @@ DuplicateScannerUtils.GetCachedDuplicatesForId = function(pClusterRecordId)
  * DuplicateScannerUtils.RefreshUnrelatedDuplicateRelations(targetEntity);<br />
  */
 DuplicateScannerUtils.RebuildDuplicatesCache = function(pFilterName, pTargetEntity,
-pQueryTargetRecords, pDuplicateFieldsConfig, pCallExternalWebserviceFunction)
+pQueryTargetRecords, pDuplicateFieldsConfig, pResultFields, pFormatValuesConsumeWebserviceCallback)
 {   
     
     logging.log("in RebuildDuplicatesCache -> ");
@@ -248,8 +254,7 @@ pQueryTargetRecords, pDuplicateFieldsConfig, pCallExternalWebserviceFunction)
      * First it gets checked if the current id has already been identified. If that's the case it'll continue with the next.
      * Otherwise an object gets build in the form of ["FilterFieldName" = "FilterFieldValueFromQuery"] with which a scan for possible duplicates get's started
      */
-    let resultFields = _DuplicateScannerUtils._loadResultFields(pFilterName, pTargetEntity);
-    logging.log("configured resultFields -> " + resultFields);
+    logging.log("configured pResultFields -> " + pResultFields);
     var duplicatesToInsertQueries = [];
     for (b = 0; b < targetRecordsData.length; b++) 
     {
@@ -273,14 +278,14 @@ pQueryTargetRecords, pDuplicateFieldsConfig, pCallExternalWebserviceFunction)
         logging.log("idValue -> " + idValue);
         
         let foundDuplicates = DuplicateScannerUtils.ScanForDuplicates(pFilterName, pTargetEntity,
-                                entityFieldValuesRay, resultFields, idField, idValue, pCallExternalWebserviceFunction, useExternalWebservice)
+                                entityFieldValuesRay, pResultFields, idField, idValue, pFormatValuesConsumeWebserviceCallback, useExternalWebservice)
         
         //        logging.log("foundDuplicates -> " + JSON.stringify(foundDuplicates));
         logging.log("foundDuplicates -> " + foundDuplicates);
-        logging.log("foundDuplicates.length -> " + foundDuplicates.length);
+        
         if(foundDuplicates == null || foundDuplicates.length == 0)
             continue;
-        
+        logging.log("foundDuplicates.length -> " + foundDuplicates.length);
  
         //Insert all found duplicate ids into an cache array because those ids don't have to be checked again lateron.
         let foundDuplicateIds = [];
@@ -305,11 +310,43 @@ pQueryTargetRecords, pDuplicateFieldsConfig, pCallExternalWebserviceFunction)
     return db.inserts(duplicatesToInsertQueries);
 }
 
-//Array
-//[DB_FELD, ENTITY_FELD, IS_ID, USE_FOR_SEARCH]
-//["CONTACTID", "CONTACTID", true, false]
-//["FIRSTNAME", "FIRSTNAME", false, true]
-DuplicateScannerUtils.LoadDuplicateIndexFieldsConfiguration = function(pFilterName, pTargetEntity)
+/* 
+ * Loads the configured resultfields as array
+ * 
+ *  @param {String} pFilterName Name of the filter
+ *  @param {String} pTargetEntity Entity which has been configured
+ *  @returns {String[]} Resultfields as array 
+ */
+DuplicateScannerUtils.LoadResultFields = function(pFilterName, pTargetEntity)
+{
+//    select dsic.DB_FIELD_NAME, dsic.ENTITY_FIELD_NAME, dsic.IS_ID_FIELD, dsic.USE_FOR_INDEX_DUPLICATE_SEARCH from DUPLICATESCANNERINDEXCONFIG dsic
+//    join DUPLICATESCANNER ds on ds.ID = dsic.DUPLICATESCANNER_ID
+//    where ds.FILTER_NAME = 'PersonDuplicates'
+//    and ds.ENTITY_TO_SCAN_NAME = 'Person_entity'
+//    order by dsic.IS_ID_FIELD desc
+
+    let duplicateResultFields = SqlCondition.begin()
+        .andPrepare("DUPLICATESCANNER.FILTER_NAME", pFilterName)
+        .andPrepare("DUPLICATESCANNER.ENTITY_TO_SCAN_NAME", pTargetEntity)
+        .buildSql("select dsrfc.ENTITY_FIELD_NAME from DUPLICATESCANNERRESULTFIELDCONFIG dsrfc join DUPLICATESCANNER on DUPLICATESCANNER.ID = dsrfc.DUPLICATESCANNER_ID"
+        , "1=2");
+    
+    logging.log("duplicateResultFields condition-> " + duplicateResultFields);
+    return db.array(db.COLUMN, duplicateResultFields);
+}
+
+/* 
+ * Loads the fields and their configuration. 
+ * One field record is in the following format: 
+ * [DB_FELD, ENTITY_FELD, IS_ID, USE_FOR_SEARCH]
+ * Example:
+ * ["FIRSTNAME", "FIRSTNAME", false, true]
+ * 
+ *  @param {String} pFilterName Name of the filter
+ *  @param {String} pTargetEntity Entity which has been configured
+ *  @returns {String[[]]} An Array of Arrays in the format described above 
+ */
+DuplicateScannerUtils.LoadIndexFieldsConfiguration = function(pFilterName, pTargetEntity)
 {
 //    select dsic.DB_FIELD_NAME, dsic.ENTITY_FIELD_NAME, dsic.IS_ID_FIELD, dsic.USE_FOR_INDEX_DUPLICATE_SEARCH from DUPLICATESCANNERINDEXCONFIG dsic
 //    join DUPLICATESCANNER ds on ds.ID = dsic.DUPLICATESCANNER_ID
@@ -327,8 +364,27 @@ DuplicateScannerUtils.LoadDuplicateIndexFieldsConfiguration = function(pFilterNa
     return db.table(duplicateIndexFieldConfigurations);
 }
 
+/* 
+ * Scans for duplicates based on the configured prefilters and the pFilterFieldValueRays.
+ * First the prefilters get applied one after another until the count of the returned data is in the allowed threshold.
+ * Then, the duplicate search using the index starts. All fields which have been configured will be used here.
+ * If the usage of an external webservice has been activated, the result will then be given to the pFormatValuesConsumeWebserviceCallback via parameter.
+ * To access the values it is advised to run thru the parameter like an array and access its value by key which is the index field name. The entity
+ * field names can be converted using DuplicateScannerUtils.TranslateEntityToIndexFields
+ * 
+ * Attention!
+ * If it's a single ScanForDuplicates call it doesn't matter what the callback returns because after the callback, no more modifications follow before
+ * returning the data.
+ * If it's inside the RebuildCache the values have to be in the same format as the parameter
+ * 
+ * @param {String} 
+ * @param {String[]} 
+ * @returns {String} 
+ * @example
+ *  
+ */
 DuplicateScannerUtils.ScanForDuplicates = function(pFilterName, pTargetEntity, pFilterFieldValueRays,
-pTargetEntityResultFields, pRecordIdFieldToIgnore, pRecordIdValueToIgnore, pCallExternalWebserviceFunction, pUseExternalWebservice)
+pTargetEntityResultFields, pRecordIdFieldToIgnore, pRecordIdValueToIgnore, pFormatValuesConsumeWebserviceCallback, pUseExternalWebservice)
 {
     let possibleDuplicates = [];
     let ignoredRecordFilter = _DuplicateScannerUtils._getIgnoreRecordFilter(pRecordIdFieldToIgnore, pRecordIdValueToIgnore, pTargetEntity);
@@ -348,8 +404,8 @@ pTargetEntityResultFields, pRecordIdFieldToIgnore, pRecordIdValueToIgnore, pCall
     possibleDuplicates = _DuplicateScannerUtils._callIndexSearch(pTargetEntity, preFilter, pFilterFieldValueRays, pTargetEntityResultFields, 100);
     possibleDuplicates = possibleDuplicates[indexsearch.HITS];
     
-    if(pUseExternalWebservice && possibleDuplicates.length > 0 && pCallExternalWebserviceFunction != null)
-        possibleDuplicates = pCallExternalWebserviceFunction.apply(this, [possibleDuplicates]);
+    if(pUseExternalWebservice && possibleDuplicates.length > 0 && pFormatValuesConsumeWebserviceCallback != null)
+        possibleDuplicates = pFormatValuesConsumeWebserviceCallback.apply(this, [possibleDuplicates]);
 
     logging.log("pTargetEntity -> " + pTargetEntity);
     logging.log("preFilter -> " + preFilter);
@@ -360,6 +416,12 @@ pTargetEntityResultFields, pRecordIdFieldToIgnore, pRecordIdValueToIgnore, pCall
     return possibleDuplicates;
 }
 
+/* 
+ * Concatenates the fields with a semicolon as separated. This can be used in a sql select.
+ * 
+ *  @param {String[]} pIndexFieldsConfig Array of Names
+ *  @returns {String} String in the style of "Value1, Value2, Value3"
+ */
 DuplicateScannerUtils.BuildSqlSelectFieldsFromFieldConfig = function(pIndexFieldsConfig)
 {
     let querySelectFields = "";
@@ -374,12 +436,30 @@ DuplicateScannerUtils.BuildSqlSelectFieldsFromFieldConfig = function(pIndexField
 }
 
 
+/* 
+ * Executes a indexsearch.lookupIndexField for eacht entity field in the parameterized array
+ * and returns it as Map.
+ *  @param {String} pEntityName ...Name of the entity
+ *  @param {String[]} pEntityFields Array of the entities Fields to translate to index fields
+ *  @returns Map-like object where (key = entity field) and (value = index field)
+ *  @example
+ *  let entityResultFields = ["LASTNAME"];
+ *  let entityIndexFields = DuplicateScannerUtils.TranslateEntityToIndexFields("Person_entity", entityResultFields);
+ *  
+ *  logging.log(entityIndexFields["LASTNAME"]);//=> "LASTNAME_value"
+ */
 DuplicateScannerUtils.TranslateEntityToIndexFields = function(pEntityName, pEntityFields)
 {
     let entityIndexFields = {};
     for (let i = 0; i < pEntityFields.length; i++) 
     {
-         let indexField = indexsearch.lookupIndexField(pEntityName, pEntityFields[i]);
+        let entityFieldName = pEntityFields[i];
+
+        //todo remove if api has been fixed
+        if(entityFieldName.startsWith(pEntityName))
+            entityFieldName = entityFieldName.replace(pEntityName + ".", "");
+
+         let indexField = indexsearch.lookupIndexField(pEntityName, entityFieldName);
          entityIndexFields[pEntityFields[i]] = indexField;
     }
     return entityIndexFields;
@@ -497,24 +577,6 @@ _DuplicateScannerUtils._isUseExternalWebservice = function(pFilterName, pTargetE
     return (isUseWebservice == 0) ? false : true;
 }
 
-_DuplicateScannerUtils._loadResultFields = function(pFilterName, pTargetEntity)
-{
-//    select dsic.DB_FIELD_NAME, dsic.ENTITY_FIELD_NAME, dsic.IS_ID_FIELD, dsic.USE_FOR_INDEX_DUPLICATE_SEARCH from DUPLICATESCANNERINDEXCONFIG dsic
-//    join DUPLICATESCANNER ds on ds.ID = dsic.DUPLICATESCANNER_ID
-//    where ds.FILTER_NAME = 'PersonDuplicates'
-//    and ds.ENTITY_TO_SCAN_NAME = 'Person_entity'
-//    order by dsic.IS_ID_FIELD desc
-
-    let duplicateResultFields = SqlCondition.begin()
-        .andPrepare("DUPLICATESCANNER.FILTER_NAME", pFilterName)
-        .andPrepare("DUPLICATESCANNER.ENTITY_TO_SCAN_NAME", pTargetEntity)
-        .buildSql("select dsrfc.ENTITY_FIELD_NAME from DUPLICATESCANNERRESULTFIELDCONFIG dsrfc join DUPLICATESCANNER on DUPLICATESCANNER.ID = dsrfc.DUPLICATESCANNER_ID"
-        , "1=2");
-    
-    logging.log("duplicateResultFields condition-> " + duplicateResultFields);
-    return db.array(db.COLUMN, duplicateResultFields);
-}
-
 /* 
  * Creates an array of arrays containing the entity field name paired with it's value.
  * 
@@ -676,6 +738,9 @@ _DuplicateScannerUtils._callIndexSearch = function(pTargetEntity, pPreFilterJson
 
     indexQuery = _DuplicateScannerUtils._setResultFields(indexQuery, pResultFields);
     
+    //indexQuery = indexQuery.addResultFields(["Person_entity.FIRSTNAME", "Person_entity.LASTNAME"]);
+    //indexQuery = indexQuery.addResultFields(["FIRSTNAME", "LASTNAME"]);
+    
     let filterPatternConfig = _DuplicateScannerUtils._buildFilterPatternConfig(pEntityFieldValueRays, pTargetEntity);
     if(filterPatternConfig != null)
     {
@@ -683,10 +748,8 @@ _DuplicateScannerUtils._callIndexSearch = function(pTargetEntity, pPreFilterJson
         indexQuery = indexQuery.addFilter(filterPatternString);
         logging.log("real filter PatternString -> " + filterPatternString);
     }
-    
     logging.log("parsedFilterAsPatternTerm -> " + parsedFilterAsPatternTerm);
     
-    
     return indexsearch.searchIndex(indexQuery);
 }