From 62ae27b720f5ebfd63ac611e946b7c6a26834fbf Mon Sep 17 00:00:00 2001
From: "d.buechler" <d.buechler@adito.de>
Date: Thu, 10 Oct 2019 14:51:26 +0200
Subject: [PATCH] Several changes to the ScanForDuplicates algorithm have been
made. It's now possible to have a scanner configuration without any
prefilters. No threshold logic is applied, only the configured fields are
used to search. On the other hand is it now possible to have a scanner with
only a prefilter.
---
.../recordcontainer/contentProcess.js | 2 +-
.../PersonDuplicatesFilter_view.aod | 1 -
process/DuplicateScanner_lib/process.js | 121 ++++++++++++------
3 files changed, 80 insertions(+), 44 deletions(-)
diff --git a/entity/Duplicates_entity/recordcontainers/recordcontainer/contentProcess.js b/entity/Duplicates_entity/recordcontainers/recordcontainer/contentProcess.js
index f97c59fc29..2e4953a02c 100644
--- a/entity/Duplicates_entity/recordcontainers/recordcontainer/contentProcess.js
+++ b/entity/Duplicates_entity/recordcontainers/recordcontainer/contentProcess.js
@@ -17,7 +17,7 @@ let selectedClusterId = vars.get("$param.ClusterId_param");
let duplicateInfosQuery = "";
let selectedId = vars.get("$local.idvalues");
-logging.log("selectedId -> " + selectedId);
+
if(selectedId)
{
/*
diff --git a/neonView/PersonDuplicatesFilter_view/PersonDuplicatesFilter_view.aod b/neonView/PersonDuplicatesFilter_view/PersonDuplicatesFilter_view.aod
index 5a4edbce1a..bc5391cf0d 100644
--- a/neonView/PersonDuplicatesFilter_view/PersonDuplicatesFilter_view.aod
+++ b/neonView/PersonDuplicatesFilter_view/PersonDuplicatesFilter_view.aod
@@ -11,7 +11,6 @@
<tableViewTemplate>
<name>PersonDuplicatesTable</name>
<favoriteActionGroup1>PersonOpenClusterDetailActionGroup</favoriteActionGroup1>
- <favoriteActionGroup2>DuplicateClusterActionGroup</favoriteActionGroup2>
<hideContentSearch v="true" />
<entityField>#ENTITY</entityField>
<isCreatable v="false" />
diff --git a/process/DuplicateScanner_lib/process.js b/process/DuplicateScanner_lib/process.js
index 582f8a7658..5b76a06e6d 100644
--- a/process/DuplicateScanner_lib/process.js
+++ b/process/DuplicateScanner_lib/process.js
@@ -1,3 +1,5 @@
+import("system.translate");
+import("ActivityTask_lib");
import("Contact_lib");
import("system.datetime");
import("JditoFilter_lib");
@@ -32,7 +34,7 @@ DuplicateScannerUtils.loadFilters = function(pFilterName, pTargetEntity)
let query = "select FILTER_CONDITION, COUNT_CHARACTERS_TO_USE, MAX_RESULTS_THRESHOLD from DUPLICATESCANNERPREFILTERCONFIG"
+ " join DUPLICATESCANNER on DUPLICATESCANNER.ID = DUPLICATESCANNERPREFILTERCONFIG.DUPLICATESCANNER_ID"
+ " where FILTER_NAME = '" + pFilterName + "'"
- + " and ENTITY_TO_SCAN_NAME = '" + pTargetEntity + "'";
+ + " and ENTITY_TO_SCAN_NAME = '" + pTargetEntity + "'";
return db.table(query);
}
@@ -148,7 +150,10 @@ DuplicateScannerUtils.CreateUnrelatedDuplicateRelation = function(pSourceContact
let newUid = util.getNewUUID();
let columns = ["ID", "SOURCEDUPLICATEID", "UNRELATEDDUPLICATEID", "CLUSTERID"];
let values = [newUid, pSourceContactId, pUnrelatedContactId, pClusterId];
-
+ logging.log("in -> CreateUnrelatedDuplicateRelation");
+ logging.log("columns -> " + columns);
+ logging.log("values -> " + values);
+
return db.insertData("UNRELATEDDUPLICATES", columns, null, values);
}
@@ -705,10 +710,17 @@ DuplicateScannerUtils.TranslateEntityToIndexFields = function(pEntityName, pEnti
}
/*
+ * Merges the source person into the target person.
+ * This
+ * - replaces the source's with the target's contactid in a predefined set of tables.
+ * - resets the standard communications of the source contact and keeps the ones of the target.
+ * - updates participants of campaigns and removes obsolet ones(which would be duplicates)
+ * - deletes the source person and contact
+ * - deletes the duplicate record, if one exists
+ * - deletes all unrelated-duplicate-relations containing the source contact id
*
- *
- * @param {String}
- * @param {String[]}
+ * @param {String} pSourceContactId The contact to be integrated into another
+ * @param {String} pTargetContactId The contact in which the source gets integrated
* @returns {String}
*/
DuplicateScannerUtils.MergePerson = function(pSourceContactId, pTargetContactId)
@@ -746,6 +758,20 @@ DuplicateScannerUtils.MergePerson = function(pSourceContactId, pTargetContactId)
return (affectedRowsCurrentAlias > 0 && deletedRows >= 2);
}
+DuplicateScannerUtils.CreateMergeSuccessActivity = function(pSourceContactId, pTargetContactId, pCurrentContactId, pContext)
+{
+ var activityDataForInsert = {
+ subject: translate.withArguments("A %0 record has been merged", [pContext]),//"Es wurde ein Personendatensatz in diesen integriert",
+ content: translate.withArguments("%0 with ID \"%1\" has been integrated into the %0 with the ID \"%2\"", [pContext, pSourceContactId, pTargetContactId]),
+ //categoryKeywordId: $KeywordRegistry.ac
+ directionKeywordId: "x",
+ responsibleContactId: pCurrentContactId
+ };
+ var activityLinks = [[pContext, pTargetContactId]];
+
+ return ActivityUtils.insertNewActivity(activityDataForInsert, activityLinks, null, db.getCurrentAlias());
+}
+
DuplicateScannerUtils.MergeOrganisation = function(pSourceContactId, pTargetContactId)
{
let updateStatements = [];
@@ -874,20 +900,37 @@ pResultFields, pRecordIdFieldToIgnore, pRecordIdValueToIgnore, pFormatValuesCons
let ignoredRecordFilter = _DuplicateScannerUtils._getIgnoreRecordFilter(pRecordIdFieldToIgnore, pRecordIdValueToIgnore, pTargetEntity);
let configuredFilters = _DuplicateScannerUtils._loadFilters(pFilterName, pTargetEntity);
- //To ensure the record which the current search is based on isnt found as result, the other configured filters get appended to
- //the filter of said records to ignore
- configuredFilters = [ignoredRecordFilter].concat(configuredFilters);
//logging.log("Found filters -> " + configuredFilters);
+ let preFilter = null;
+
+ //Only run the prefilter if filters have been configured. If not, run the indexsearch based on the field configuration
+ if(configuredFilters != null && configuredFilters.length > 0)
+ {
+ //To ensure the record which the current search is based on isnt found as result, the other configured filters get appended to
+ //the filter of said records to ignore
+ configuredFilters = [ignoredRecordFilter].concat(configuredFilters);
- let preFilter = _DuplicateScannerUtils._applyPreFilter(pTargetEntity, configuredFilters, pFilterFieldValueRays);
-
- //logging.log("preFilter welcher Elemente im erlaubten bereich ausgibt -> " + preFilter);
+ preFilter =_DuplicateScannerUtils._applyPreFilter(pTargetEntity, configuredFilters, pFilterFieldValueRays);
- if(preFilter == null)
+ //logging.log("preFilter welcher Elemente im erlaubten bereich ausgibt -> " + preFilter);
+
+ //The scan can be executed even without any prefilters. If a prefilter has been configured but doesn't match the
+ //threshold criteria no search shall be run.
+ if(preFilter == null)
+ return null;
+ }
+
+ //No prefilter and no filterfields => No indexsearch
+ if(preFilter == null && pFilterFieldValueRays.length < 1)
return null;
-
+
+ //If at this point the prefilter is null but a search has to be executed, add the ignorefilter manually that the search doesn't find the base record as duplicate to itself.
+ //This is the case if no prefilter but indexfields are configured.
+ if(preFilter == null)
+ preFilter = ignoredRecordFilter;
+
possibleDuplicates = _DuplicateScannerUtils._callIndexSearch(pTargetEntity, preFilter, pFilterFieldValueRays, pResultFields, 100);
//logging.log("possibleDuplicates -> " + JSON.stringify(possibleDuplicates));
@@ -1051,40 +1094,34 @@ _DuplicateScannerUtils._applyPreFilter = function(pTargetEntity, pFilterCountCha
*/
_DuplicateScannerUtils._callIndexSearch = function(pTargetEntity, pPreFilterJson, pEntityFieldValueRays, pResultFields, pResultSetRows)
{
- let parsedFilterAsPatternTerm = indexsearch.buildQueryFromSearchCondition(pPreFilterJson);
- ////logging.log("pTargetEntity -> " + pTargetEntity);
- //logging.log("pResultFields -> " + pResultFields);
- ////logging.log("pResultSetRows -> " + pResultSetRows);
+ let indexPattern = null;
+ let filterPattern = null;
+
+ //The pPreFilterJson is never null because it always contains at least the default ignore record filter
+ indexPattern = indexsearch.buildQueryFromSearchCondition(pPreFilterJson);
+
+ let filterPatternConfig = _DuplicateScannerUtils._buildFilterPatternConfig(pEntityFieldValueRays, pTargetEntity);
+
+ if(filterPatternConfig != null)
+ filterPattern = indexsearch.buildPatternString(filterPatternConfig);
+
+ //The indexPattern can't be null because it is required to run the search.
+ if(indexPattern == null)
+ return null;
+
let indexQuery = indexsearch.createIndexQuery()
- .setPattern(parsedFilterAsPatternTerm)
+ .setPattern(indexPattern)
.setEntities([pTargetEntity])
//.addSearchFields("Person_entity.FIRSTNAME", "Person_entity.LASTNAME", "Person_entity.CONTACTID")
//.setRows(pResultSetRows);
-
indexQuery = _DuplicateScannerUtils._setResultFields(indexQuery, pResultFields);
- //indexQuery = indexQuery.addResultFields(["Person_entity.FIRSTNAME", "Person_entity.LASTNAME"]);
- //indexQuery = indexQuery.addResultFields(["FIRSTNAME", "LASTNAME"]);
-
- let filterPatternConfig = _DuplicateScannerUtils._buildFilterPatternConfig(pEntityFieldValueRays, pTargetEntity);
- if(filterPatternConfig != null)
- {
- let filterPatternString = indexsearch.buildPatternString(filterPatternConfig);
- indexQuery = indexQuery.addFilter(filterPatternString);
- //logging.log("real filter PatternString -> " + filterPatternString);
- }
- //logging.log("parsedFilterAsPatternTerm -> " + parsedFilterAsPatternTerm);
-
- if(filterPatternConfig == null && pEntityFieldValueRays.length > 0)
- {
- //logging.log("FilterPattern ist null aber es gibt pEntityFieldValueRays -> Die Felder sollten genutzt werden, beinhalten aber keine Werte");
- return null;
- }
- else
- {
- //logging.log("Starte Indexsuche -> ");
- return indexsearch.searchIndex(indexQuery);
- }
+ if(filterPattern != null)
+ indexQuery = indexQuery.addFilter(filterPattern);
+ logging.log("indexQuery.getPattern -> " + indexQuery.getPattern());
+ logging.log("indexQuery.getFilters -> " + indexQuery.getFilters());
+ //logging.log("Starte Indexsuche -> ");
+ return indexsearch.searchIndex(indexQuery);
}
/*
@@ -1136,7 +1173,7 @@ _DuplicateScannerUtils._setResultFields = function(pIndexQuery, pResultFields)
*
* @param {String[[]]} pEntityFieldValueRays Array of Arrays containing the name of a used field and its value.
* @param {String} pTargetEntity Entity which has been configured
- * @returns {PatternConfig} PatternConfig created with "indexsearch.createPatternConfig()"
+ * @returns {PatternConfig} PatternConfig created with "indexsearch.createPatternConfig()", null if the creation wasn't successful
*/
_DuplicateScannerUtils._buildFilterPatternConfig = function(pEntityFieldValueRays, pTargetEntity)
{
--
GitLab