Fix in JditoFilter_lib. The function "JditoFilterUtils.getFilterFields" now...

Fix in JditoFilter_lib. The function "JditoFilterUtils.getFilterFields" now correctly reads fields from a filter json
The call of the index search can now correctly use filters to make use of the real duplicate search. The dynamic reading of fields comes next

entity/DuplicateScanner_entity/entityfields/testactiongroup/children/testduplicatescanner/onActionProcess.js

@@ -11,62 +11,115 @@ import("JditoFilter_lib");
 //for (let i = 0; i < filters.length; i++) 
 //{
 //    logging.log("filters[i] -> " + filters[i]);
-//    let filter = JSON.parse(filters[i]).filter;
-//    let fields = JditoFilterUtils.getFilterFields(filter.childs)
-//    testFields.concat(fields);
+//    let filter = JSON.parse(filters[i][0]).filter;
+//    let fields = JditoFilterUtils.getFilterFields(filter.childs);
+//    testFields = testFields.concat(fields);
 //}
 //logging.log("testFields -> " + testFields);
 
 
+//##############################Test Duplicate Scan######################################################
 
+//var filterName = "PersonDuplicates";
+//var targetEntity = "Person_entity";
+//var resultFieldsIdFieldName = "CONTACTID";
+//var queryPersonContactIds = "select CONTACTID, FIRSTNAME, LASTNAME, GENDER from CONTACT"
+//                            + " join PERSON on PERSONID = PERSON_ID";
+//var tmpFieldsInFilterRay = ["CONTACTID", "FIRSTNAME", "LASTNAME", "GENDER"];
+//
+//var filterFieldValueRays = [["CONTACTID", "29271db0-4253-46c9-b7c2-5e25376b9d19"], ["FIRSTNAME", "Narkus"], ["LASTNAME", "Bltinger"], ["GENDER", "m"]];
+//
+////DuplicateScannerUtils.ScanForDuplicatesIndex = function(pFilterName, pTargetEntity, pFilterFieldValueRays,
+////pTargetEntityResultFields, pRecordIdFieldToIgnore, pRecordIdValueToIgnore)
+//
+//
+//DuplicateScannerUtils.ScanForDuplicatesIndex(filterName, targetEntity, 
+//filterFieldValueRays, [], resultFieldsIdFieldName, "29271db0-4253-46c9-b7c2-5e25376b9d19");
 
-var filterName = "PersonDuplicates";
-var targetEntity = "Person_entity";
-var resultFieldsIdFieldName = "CONTACTID";
-var queryPersonContactIds = "select CONTACTID, FIRSTNAME, LASTNAME, GENDER from CONTACT"
-                            + " join PERSON on PERSONID = PERSON_ID";
-var tmpFieldsInFilterRay = ["CONTACTID", "FIRSTNAME", "LASTNAME", "GENDER"];
-
-var filterFieldValueRays = [["CONTACTID", "29271db0-4253-46c9-b7c2-5e25376b9d19"], ["FIRSTNAME", "Narkus"], ["LASTNAME", "Bltinger"], ["GENDER", "M"]];
-
-//DuplicateScannerUtils.ScanForDuplicatesIndex = function(pFilterName, pTargetEntity, pFilterFieldValueRays,
-//pTargetEntityResultFields, pRecordIdFieldToIgnore, pRecordIdValueToIgnore)
-
-
-DuplicateScannerUtils.ScanForDuplicatesIndex(filterName, targetEntity, 
-filterFieldValueRays, [], resultFieldsIdFieldName, "29271db0-4253-46c9-b7c2-5e25376b9d19");
 
+//####################################################################################
 
 
-//let indexQuery = indexsearch.createIndexQuery()
-//                            .setPattern("(+(-contactid_value:(29271db0-4253-46c9-b7c2-5e25376b9d19)))")
-//                            .setEntities("Person_entity")
-//                            .addResultIndexFields(indexsearch.FIELD_ID)
-//                            .addResultFields([])
-//                            //.addSearchFields("Person_entity.FIRSTNAME", "Person_entity.LASTNAME", "Person_entity.CONTACTID");
-//                            //.setRows(100);
+//logging.log("TEST INDEX API with Entities");
+//logging.log(indexsearch.lookupIndexField("Person_entity", "FIRSTNAME"));
+//logging.log(indexsearch.lookupIndexField("Person_entity", "FIRSTNAME.value"));
+//logging.log(indexsearch.lookupIndexField("Person_entity", "PersAddresses.CITY"));
+//logging.log(indexsearch.lookupIndexField("Person_entity", "PersAddresses.CITY.value"));
+//logging.log(indexsearch.lookupIndexField(null, "Person_entity.FIRSTNAME.value"));
+//logging.log(indexsearch.lookupIndexField(null, "Person_entity.PersAddresses.CITY.value"));
+//var json = '{"entity":"Person_entity","filter":{"type":"group","operator":"AND","childs":[{"type":"row","name":"FIRSTNAME","operator":"STARTSWITH","value":"asd","key":"","contenttype":"TEXT"},{"type":"group","operator":"OR","childs":[{"type":"row","name":"LASTNAME","operator":"STARTSWITH","value":"L","key":"","contenttype":"TEXT"}]}]}}';
+//logging.log(indexsearch.buildQueryFromSearchCondition(json));
 //
-//var res = indexsearch.searchIndex(indexQuery);
-//logging.log("" + res);
-
-
 //var t1 = indexsearch.createTerm("lisa").setEntityField("Person_entity.FIRSTNAME");
-//var t3 = indexsearch.createWildcardTerm("L").setEntityField("Person_entity.LASTNAME");
+//var t2 = indexsearch.createTerm("sommer").setEntityField("Person_entity.LASTNAME");
+//var t3 = indexsearch.createWildcardTerm("L").setEntityField("Person_entity.PersAddresses.CITY");
 //
-//var patternConf = indexsearch.createPatternConfig().or(t1).or(t3);
+//var patternConf = indexsearch.createPatternConfig().or(t1).or(t2).or(t3);
 //var pattern = indexsearch.buildPatternString(patternConf);
-//logging.log(pattern);
+//logging.log(pattern); 
 //
 //var query = indexsearch.createIndexQuery()
-//.setPattern("(+(-contactid_value:(29271db0-4253-46c9-b7c2-5e25376b9d19)))")
+//.setPattern(pattern)
 //.setEntities("Person_entity")
-//.addResultIndexFields(indexsearch.FIELD_ID)
-//.addResultFields("Person_entity.FIRSTNAME");
-////.addSearchFields("Person_entity.FIRSTNAME", "Person_entity.LASTNAME");
+////.addResultIndexFields(indexsearch.FIELD_ID)
+//.addResultFields("Person_entity.FIRSTNAME")
+//.addSearchFields("Person_entity.FIRSTNAME", "Person_entity.LASTNAME");
 //
 //var res = indexsearch.searchIndex(query);
 //logging.log("" + res);
 
+
+
+//####################################################################################
+
+
+
+
+
+let indexQuery = indexsearch.createIndexQuery()
+                            .setPattern("(+(-contactid_value:(29271db0-4253-46c9-b7c2-5e25376b9d19)) +gender_value:m*)")
+                            .setEntities(["Person_entity"])
+                            .addResultFields("Person_entity.FIRSTNAME")
+                            .setRows(50);
+
+
+let filterTerm1 = indexsearch.createTerm("Narkus")
+                            .setIndexField("firstname_value")
+                            .setFuzzySearchFactor(0);
+let filterTerm2 = indexsearch.createTerm("Bltinger")
+                            .setIndexField("lastname_value")
+                            .setFuzzySearchFactor(0);
+
+let filterPatternConfig = indexsearch.createPatternConfig().and(filterTerm1);
+
+let filterPatternString = indexsearch.buildPatternString(filterPatternConfig);
+logging.log("Hauptsuche filterPatternString -> " + filterPatternString);
+indexQuery = indexQuery.addFilter(filterPatternString);
+
+let searchResult = indexsearch.searchIndex(indexQuery);
+logging.log("searchResult -> " + searchResult);
+
+logging.log("searchResults hits length -> " + searchResult[indexsearch.HITS].length);
+
+for (let i = 0; i < searchResult[indexsearch.HITS].length; i++) 
+{
+    logging.log("Treffer Nr -> " + i);
+    //searchResults hits 0 -> {#ADITO_SEARCH_ID=1868bd3a-05af-4b7f-a633-e3aec50ac45c, _index_group_=Person, #ADITO_SEARCH_TYPE=Person, firstname_value=Peter, _local_id_=1868bd3a-05af-4b7f-a633-e3aec50ac45c}
+    let localId = searchResult[indexsearch.HITS][i]["_local_id_"];
+    let firstname = searchResult[indexsearch.HITS][i]["firstname_value"];
+    let indexGroup = searchResult[indexsearch.HITS][i]["_index_group_"];
+    logging.log("localId -> " + localId);
+    logging.log("firstname -> " + firstname);
+    logging.log("indexGroup -> " + indexGroup);
+}
+
+
+
+
+
+//####################################################################################
+
+
 //
 //var filterName = "PersonDuplicates";
 //var targetEntity = "Person_entity";

entity/Person_entity/Person_entity.aod

@@ -1365,6 +1365,10 @@ Usually this is used for filtering COMMUNICATION-entries by a specified contact
             <element>addr</element>
           </additionalFieldNameAliases>
         </indexRecordFieldMapping>
+        <indexRecordFieldMapping>
+          <name>GENDER.value</name>
+          <indexFieldType>PHONETIC_NAME</indexFieldType>
+        </indexRecordFieldMapping>
       </indexFieldMappings>
     </indexRecordContainer>
   </recordContainers>

entity/Person_entity/recordcontainers/index/query.js

@@ -27,6 +27,7 @@ sqlQuery = 'select CONTACT.CONTACTID as "_uid_" '
     + ", PERSON.FIRSTNAME "
     + ", PERSON.LASTNAME "
     + ", PERSON.SALUTATION "
+    + ", PERSON.GENDER "
     + ', PERSON.TITLE as "TITLE.value"'
     + ", CONTACT.CONTACTID "
     + ", CONTACT.ORGANISATION_ID "

process/DuplicateScanner_lib/process.js

@@ -436,12 +436,12 @@ pTargetEntityResultFields, pRecordIdFieldToIgnore, pRecordIdValueToIgnore)
         logging.log("pFilterFieldValueRays -> " + pFilterFieldValueRays);
         logging.log("pRecordIdFieldToIgnore -> " + pRecordIdFieldToIgnore);
         
+        //(pTargetEntity, pPreFilterJson, pEntityFieldValueRays, pResultIndexFields, pResultFields, pResultSetRows)
         //run actual index duplicate search
-        possibleDuplicates = DuplicateScannerUtils._callIndexSearch(pTargetEntity, preFilter, pFilterFieldValueRays, [pRecordIdFieldToIgnore], 100)
+        possibleDuplicates = DuplicateScannerUtils._callIndexSearch(pTargetEntity, preFilter, pFilterFieldValueRays, [indexsearch.FIELD_ID], ["Person_entity.FIRSTNAME"], 100);
+        logging.log("possibleDuplicates -> " + possibleDuplicates);
     }
     
-
-    
     return possibleDuplicates;
 }
 
@@ -505,8 +505,28 @@ DuplicateScannerUtils._applyPreFilterIndex = function(pTargetEntity, pFilterCoun
         
         //Workaround to load the smallest possible resultset because only the TOTALHITS are relevant at this time
         //Only load "indexsearch.FIELD_ID" and a resultSet size of 1
-        let totalHits = DuplicateScannerUtils._callIndexSearch(pTargetEntity, JSON.stringify(filter), [], 
-        [indexsearch.FIELD_ID], ["Person_entity.FIRSTNAME"], 1)[indexsearch.TOTALHITS];
+        let searchResult = DuplicateScannerUtils._callIndexSearch(pTargetEntity, JSON.stringify(filter), [], 
+        [indexsearch.FIELD_ID], ["Person_entity.FIRSTNAME"], 1);
+        logging.log("searchResults hits length -> " + searchResult[indexsearch.HITS].length);
+        logging.log("searchResults hits length -> " + searchResult[indexsearch.HITS][0]);
+        
+        if(searchResult[indexsearch.HITS].length < 80)
+        {
+            for (let i = 0; i < searchResult[indexsearch.HITS].length; i++) 
+            {
+                logging.log("Treffer Nr -> " + i);
+                //searchResults hits 0 -> {#ADITO_SEARCH_ID=1868bd3a-05af-4b7f-a633-e3aec50ac45c, _index_group_=Person, #ADITO_SEARCH_TYPE=Person, firstname_value=Peter, _local_id_=1868bd3a-05af-4b7f-a633-e3aec50ac45c}
+                let localId = searchResult[indexsearch.HITS][i]["_local_id_"];
+                let firstname = searchResult[indexsearch.HITS][i]["firstname_value"];
+                let indexGroup = searchResult[indexsearch.HITS][i]["_index_group_"];
+                logging.log("localId -> " + localId);
+                logging.log("firstname -> " + firstname);
+                logging.log("indexGroup -> " + indexGroup);
+            }
+        }
+        
+        let totalHits = searchResult[indexsearch.TOTALHITS]
+        
         
         logging.log("totalHits -> " + totalHits);
         
@@ -530,7 +550,7 @@ DuplicateScannerUtils._applyPreFilterIndex = function(pTargetEntity, pFilterCoun
             //let resultRows = entities.getRows(loadRowsConfig);
             
             
-            logging.log("Im return valider Filter  -> " + combinedFilter);
+            logging.log("Im return valider Filter  -> " + JSON.stringify(filter));
             return JSON.stringify(filter);
         }
     }
@@ -541,7 +561,6 @@ DuplicateScannerUtils._applyPreFilterIndex = function(pTargetEntity, pFilterCoun
 DuplicateScannerUtils._callIndexSearch = function(pTargetEntity, pPreFilterJson, pEntityFieldValueRays, pResultIndexFields, pResultFields, pResultSetRows)
 {
     let parsedFilterAsPatternTerm = indexsearch.buildQueryFromSearchCondition(pPreFilterJson);
-    logging.log("parsedFilterAsPatternTerm -> " + parsedFilterAsPatternTerm);
     logging.log("pTargetEntity -> " + pTargetEntity);
     logging.log("pResultIndexFields -> " + pResultIndexFields);
     logging.log("pResultFields -> " + pResultFields);
@@ -549,29 +568,39 @@ DuplicateScannerUtils._callIndexSearch = function(pTargetEntity, pPreFilterJson,
     let indexQuery = indexsearch.createIndexQuery()
                                 .setPattern(parsedFilterAsPatternTerm)
                                 .setEntities([pTargetEntity])
-                                .addResultIndexFields(pResultIndexFields)
+                                //.addResultIndexFields(pResultIndexFields)
                                 .addResultFields(pResultFields)
                                 //.addSearchFields("Person_entity.FIRSTNAME", "Person_entity.LASTNAME", "Person_entity.CONTACTID")
                                 .setRows(pResultSetRows);
 
+        logging.log("pEntityFieldValueRays.length -> " + pEntityFieldValueRays.length);
     if(pEntityFieldValueRays.length > 0)
     {
         let filterPatternConfig = indexsearch.createPatternConfig();
         for (let i = 0; i < pEntityFieldValueRays.length; i++) 
         {
-            let entityFieldValue = pEntityFieldValueRays[1];
-            let entityFieldName = pEntityFieldValueRays[0];
+            let entityFieldValue = pEntityFieldValueRays[i][1];
+            let entityFieldName = pEntityFieldValueRays[i][0];
+            
+            logging.log("entityFieldValue -> " + entityFieldValue);
+            logging.log("entityFieldName -> " + entityFieldName);
+            
+            if(entityFieldName == "CONTACTID" || entityFieldName == "GENDER")
+                continue;
             
-            let indexField = indexSearch.lookupIndexField(pTargetEntity, entityFieldName);
+            let indexField = indexsearch.lookupIndexField(pTargetEntity, entityFieldName);
             
+            logging.log("indexField -> " + indexField);
             var filterTerm = indexsearch.createTerm(entityFieldValue)
                                         .setIndexField(indexField)
                                         .setFuzzySearchFactor(0);
                                         
             filterPatternConfig.and(filterTerm);
         }
+        logging.log("hallo -> ");
+        let filterPatternString = indexsearch.buildPatternString(filterPatternConfig);
         
-        let filterPatternString = indexSearch.buildPatternString(filterPatternConfig);
+        logging.log("parsedFilterAsPatternTerm -> " + parsedFilterAsPatternTerm);
         logging.log("filterPatternString -> " + filterPatternString);
         indexQuery.addFilter(filterPatternString);
     }
@@ -970,7 +999,7 @@ _DuplicateScannerUtils._insertValuesInFilterTemplate = function(pJsonRootNode, p
     for(var filterChildNode in pJsonRootNode)
     {
         var currentNode = pJsonRootNode[filterChildNode];
-        logging.log("currentNode -> " + JSON.stringify(currentNode));
+//        logging.log("currentNode -> " + JSON.stringify(currentNode));
         if(currentNode.type == "row")
         {
             
@@ -978,7 +1007,7 @@ _DuplicateScannerUtils._insertValuesInFilterTemplate = function(pJsonRootNode, p
             let fieldValue = pEntitiyFieldAndValueMap[fieldName];
             pCountCharsOfValueToUse = parseInt(pCountCharsOfValueToUse, 10);
             
-            logging.log("fieldValue -> " + JSON.stringify(pEntitiyFieldAndValueMap));
+//            logging.log("fieldValue -> " + JSON.stringify(pEntitiyFieldAndValueMap));
             if(fieldValue == null)
             {
                 logging.show("Duplicate Scan: Requested value for field " + fieldName + " not present in the provided valueslist");
@@ -995,7 +1024,7 @@ _DuplicateScannerUtils._insertValuesInFilterTemplate = function(pJsonRootNode, p
                 && _DuplicateScannerUtils._isValueLongerThanCharsToUse(fieldValue.length, pCountCharsOfValueToUse))
             {
                 fieldValue = fieldValue.substring(0, pCountCharsOfValueToUse);
-                logging.log("fieldValue geschnitten -> " + fieldValue);
+//                logging.log("fieldValue geschnitten -> " + fieldValue);
             }
             
             pJsonRootNode[filterChildNode].value = fieldValue;
@@ -1003,9 +1032,9 @@ _DuplicateScannerUtils._insertValuesInFilterTemplate = function(pJsonRootNode, p
         else
         {
             //currentNode.type == "group"
-            logging.log("type == group -> ");
+//            logging.log("type == group -> ");
             let populatedChildNodes = _DuplicateScannerUtils._insertValuesInFilterTemplate(currentNode.childs, pEntitiyFieldAndValueMap, pCountCharsOfValueToUse);
-            logging.log("populatedChildNodes -> " + JSON.stringify(populatedChildNodes));
+//            logging.log("populatedChildNodes -> " + JSON.stringify(populatedChildNodes));
             pJsonRootNode[filterChildNode].childs = populatedChildNodes;
         }
     }

process/JditoFilter_lib/process.js

@@ -108,8 +108,7 @@ JditoFilterUtils.getFilterFields = function (pFilterJsonRootNode)
     let filterFields = [];
     for(var filterChildNode in pFilterJsonRootNode)
     {
-         var currentNode = pJsonRootNode[filterChildNode];
-        
+         var currentNode = pFilterJsonRootNode[filterChildNode];
         if(currentNode.type == "row")
         {
             let fieldName = currentNode.name;