Removed the block size configuration from the client. Default value is 5000....

Removed the block size configuration from the client. Default value is 5000. All scanners now use the same value, it can be changed via designer custom setting in the preferences
2 fields in the person entity have been added, now the firstname and the lastname fields exist as phonetic and text_no_stopwords
Refactorings

.liquibase/Data_alias/basic/2019.2.1/Duplicates/create_duplicateScanner.xml

@@ -14,9 +14,6 @@
             <column name="EXTERNAL_SERVICE_USAGE_ALLOWED" type="INTEGER" >
                 <constraints nullable="false"/>
             </column>
-            <column name="BLOCK_SIZE" type="INTEGER">
-                <constraints nullable="false"/>
-            </column>
             <column name="USER_NEW" type="NVARCHAR(50)">
                 <constraints nullable="false"/>
             </column>

entity/DuplicateScanner_entity/DuplicateScanner_entity.aod

@@ -120,10 +120,6 @@
         </entityActionField>
       </children>
     </entityActionGroup>
-    <entityField>
-      <name>BLOCK_SIZE</name>
-      <title>Data block size</title>
-    </entityField>
   </entityFields>
   <recordContainers>
     <dbRecordContainer>
@@ -172,10 +168,6 @@
           <name>DATE_EDIT.value</name>
           <recordfield>DUPLICATESCANNER.DATE_EDIT</recordfield>
         </dbRecordFieldMapping>
-        <dbRecordFieldMapping>
-          <name>BLOCK_SIZE.value</name>
-          <recordfield>DUPLICATESCANNER.BLOCK_SIZE</recordfield>
-        </dbRecordFieldMapping>
       </recordFieldMappings>
     </dbRecordContainer>
   </recordContainers>

entity/DuplicateScanner_entity/entityfields/runactiongroup/children/rebuildorganisationduplicatescache/onActionProcess.js

@@ -3,18 +3,18 @@ import("DuplicateScanner_lib");
 
 var filterName = "OrganisationDuplicates";
 var targetEntity = "Organisation_entity";
-var recordBlockSize = DuplicateScannerUtils.GetBlockSizeForScanner(filterName, targetEntity);
+var recordBlockSize = DuplicateScannerUtils.GetBlockSize();
 let duplicateFieldsConfig = DuplicateScannerUtils.LoadIndexFieldsConfiguration(filterName, targetEntity);
 let resultFields = DuplicateScannerUtils.LoadResultFields(filterName, targetEntity);
 
-//logging.log("duplicateFieldsConfig -> " + duplicateFieldsConfig);
-//logging.log("resultFields -> " + resultFields);
-
-logging.log("Delete all " + targetEntity + " duplicates");
+logging.log(filterName + ": Delete duplicates -> ");
 DuplicateScannerUtils.DeleteDuplicateClustersByTargetEntity(targetEntity);
 
-logging.log("Recalculate all " + targetEntity + " duplicates");
+logging.log(filterName + ": Recalculate duplicates -> ");
 DuplicateScannerUtils.RebuildDuplicatesCache(filterName, targetEntity, recordBlockSize,
 duplicateFieldsConfig, resultFields, null);
 
-DuplicateScannerUtils.RefreshUnrelatedDuplicateRelations(targetEntity);
\ No newline at end of file
+logging.log(filterName + ": Refresh unrelated duplicates -> ");
+DuplicateScannerUtils.RefreshUnrelatedDuplicateRelations(targetEntity);
+
+logging.log(filterName + ": Done rebuilding ");
\ No newline at end of file

entity/DuplicateScanner_entity/entityfields/runactiongroup/children/rebuildpersonduplicatescache/onActionProcess.js

-import("system.util");
-import("Employee_lib");
-import("system.process");
+import("system.project");
 import("system.logging");
 import("DuplicateScanner_lib");
 
-//let processParameters = {
-//    filterName: "PersonDuplicates",
-//    targetEntity: "Person_entity" //process.execute is only able to handle strings
-//}
-//let userId = EmployeeUtils.getCurrentUserId();
-//
-//try 
-//{
-//    let processId = "manualrun_rebuild_duplicatecache_" + util.getNewUUID();
-//    process.executeTimer(processId, "RebuildDuplicatesCache_serverProcess", 0, true, false, process.TIMERTYPE_SERVER_RUN, userId, false, process.THREADPRIORITY_LOW)
-//    process.stopTimer(processId);
-//} 
-//catch (exception) 
-//{ 
-//    logging.log(" exception-> " + exception);
-//}
-
-
 var filterName = "PersonDuplicates";
 var targetEntity = "Person_entity";
-var recordBlockSize = DuplicateScannerUtils.GetBlockSizeForScanner(filterName, targetEntity);
+var recordBlockSize = DuplicateScannerUtils.GetBlockSize();
+logging.log("recordBlockSize -> " + recordBlockSize);
+
 let duplicateFieldsConfig = DuplicateScannerUtils.LoadIndexFieldsConfiguration(filterName, targetEntity);
 let resultFields = DuplicateScannerUtils.LoadResultFields(filterName, targetEntity);
 
-//logging.log("duplicateFieldsConfig -> " + duplicateFieldsConfig);
-//logging.log("resultFields -> " + resultFields);
-
-logging.log("Delete all " + targetEntity + " duplicates");
+logging.log(filterName + ": Delete duplicates -> ");
 DuplicateScannerUtils.DeleteDuplicateClustersByTargetEntity(targetEntity);
 
-logging.log("Recalculate all " + targetEntity + " duplicates");
+logging.log(filterName + ": Recalculate duplicates -> ");
 DuplicateScannerUtils.RebuildDuplicatesCache(filterName, targetEntity, recordBlockSize,
 duplicateFieldsConfig, resultFields, null);
 
+logging.log(filterName + ": Refresh unrelated duplicates -> ");
 DuplicateScannerUtils.RefreshUnrelatedDuplicateRelations(targetEntity);
 
+logging.log(filterName + ": Done rebuilding ");
\ No newline at end of file

entity/Person_entity/Person_entity.aod

@@ -1082,30 +1082,32 @@ Usually this is used for filtering COMMUNICATION-entries by a specified contact
       <mandatory v="false" />
     </entityParameter>
     <entityField>
-      <name>STANDARD_ZIP</name>
-      <title>Standard zip</title>
-      <valueProcess>%aditoprj%/entity/Person_entity/entityfields/standard_zip/valueProcess.js</valueProcess>
+      <name>IndexCommunication</name>
     </entityField>
     <entityField>
-      <name>STANDARD_STREET</name>
-      <title>Standard street</title>
-      <valueProcess>%aditoprj%/entity/Person_entity/entityfields/standard_street/valueProcess.js</valueProcess>
+      <name>IndexAddress</name>
     </entityField>
     <entityField>
-      <name>STANDARD_CITY</name>
-      <title>Standard city</title>
-      <valueProcess>%aditoprj%/entity/Person_entity/entityfields/standard_city/valueProcess.js</valueProcess>
+      <name>IndexCountry</name>
     </entityField>
     <entityField>
-      <name>STANDARD_MAIL</name>
-      <title>Standard mail</title>
-      <valueProcess>%aditoprj%/entity/Person_entity/entityfields/standard_mail/valueProcess.js</valueProcess>
+      <name>IndexZIP</name>
     </entityField>
     <entityField>
-      <name>STANDARD_PHONE</name>
-      <title>Standard phone</title>
-      <valueProcess>%aditoprj%/entity/Person_entity/entityfields/standard_phone/valueProcess.js</valueProcess>
+      <name>IndexCity</name>
     </entityField>
+    <entityField>
+      <name>STANDARD_COUNTRY</name>
+      <valueProcess>%aditoprj%/entity/Person_entity/entityfields/standard_country/valueProcess.js</valueProcess>
+    </entityField>
+    <entityActionField>
+      <name>ActionField</name>
+      <onActionProcess>%aditoprj%/entity/Person_entity/entityfields/actionfield/onActionProcess.js</onActionProcess>
+    </entityActionField>
+    <entityParameter>
+      <name>OnlyOwnSupervised_param</name>
+      <expose v="true" />
+    </entityParameter>
   </entityFields>
   <recordContainers>
     <dbRecordContainer>
@@ -1348,6 +1350,10 @@ Usually this is used for filtering COMMUNICATION-entries by a specified contact
           <isFilterable v="true" />
           <filtertype>BASIC</filtertype>
         </consumerMapping>
+        <dbRecordFieldMapping>
+          <name>STANDARD_COUNTRY.value</name>
+          <recordfield>ADDRESS.COUNTRY</recordfield>
+        </dbRecordFieldMapping>
       </recordFieldMappings>
       <filterExtensions>
         <filterExtensionSet>
@@ -1364,7 +1370,6 @@ Usually this is used for filtering COMMUNICATION-entries by a specified contact
       <configMode>INDEXGROUP_DEFINITION</configMode>
       <indexRecordAlias>Data_alias</indexRecordAlias>
       <query>%aditoprj%/entity/Person_entity/recordcontainers/index/query.js</query>
-      <subProcess>%aditoprj%/entity/Person_entity/recordcontainers/index/subProcess.js</subProcess>
       <affectedTables>
         <element>ADDRESS</element>
         <element>COMMUNICATION</element>
@@ -1373,12 +1378,6 @@ Usually this is used for filtering COMMUNICATION-entries by a specified contact
         <element>ORGANISATION</element>
       </affectedTables>
       <affectedIds>%aditoprj%/entity/Person_entity/recordcontainers/index/affectedIds.js</affectedIds>
-      <includedConsumers>
-        <element>PersAddresses</element>
-        <element>OrgAddresses</element>
-        <element>Communications</element>
-      </includedConsumers>
-      <affectedConsumerIds>%aditoprj%/entity/Person_entity/recordcontainers/index/affectedConsumerIds.js</affectedConsumerIds>
       <indexFieldMappings>
         <indexRecordFieldMapping>
           <name>FIRSTNAME.value</name>
@@ -1415,43 +1414,48 @@ Usually this is used for filtering COMMUNICATION-entries by a specified contact
           <indexFieldType>STRING</indexFieldType>
         </indexRecordFieldMapping>
         <indexRecordFieldMapping>
-          <name>STANDARD_EMAIL_COMMUNICATION.value</name>
-          <indexFieldType>EMAIL</indexFieldType>
-          <additionalFieldNameAliases>
-            <element>addr</element>
-          </additionalFieldNameAliases>
-          <isIndexed v="true" />
+          <name>GENDER.value</name>
+          <indexFieldType>STRING</indexFieldType>
         </indexRecordFieldMapping>
         <indexRecordFieldMapping>
-          <name>STANDARD_PHONE_COMMUNICATION.value</name>
-          <indexFieldType>TELEPHONE</indexFieldType>
+          <name>IndexAddress.value</name>
+          <indexFieldType>ADDRESS</indexFieldType>
           <additionalFieldNameAliases>
-            <element>addr</element>
+            <element>address</element>
           </additionalFieldNameAliases>
+          <isMultiValued v="true" />
         </indexRecordFieldMapping>
         <indexRecordFieldMapping>
-          <name>GENDER.value</name>
-          <indexFieldType>PHONETIC_NAME</indexFieldType>
-        </indexRecordFieldMapping>
-        <indexRecordFieldMapping>
-          <name>STANDARD_CITY.value</name>
-          <indexFieldType>TEXT_NO_STOPWORDS</indexFieldType>
-        </indexRecordFieldMapping>
-        <indexRecordFieldMapping>
-          <name>STANDARD_MAIL.value</name>
-          <indexFieldType>EMAIL</indexFieldType>
+          <name>IndexCity.value</name>
+          <indexFieldType>ADDRESS</indexFieldType>
+          <additionalFieldNameAliases>
+            <element>city</element>
+          </additionalFieldNameAliases>
+          <isMultiValued v="true" />
         </indexRecordFieldMapping>
         <indexRecordFieldMapping>
-          <name>STANDARD_PHONE.value</name>
-          <indexFieldType>TELEPHONE</indexFieldType>
+          <name>IndexCommunication.value</name>
+          <indexFieldType>COMMUNICATION</indexFieldType>
+          <additionalFieldNameAliases>
+            <element>comm</element>
+          </additionalFieldNameAliases>
+          <isMultiValued v="true" />
         </indexRecordFieldMapping>
         <indexRecordFieldMapping>
-          <name>STANDARD_STREET.value</name>
-          <indexFieldType>TEXT_NO_STOPWORDS</indexFieldType>
+          <name>IndexCountry.value</name>
+          <indexFieldType>ADDRESS</indexFieldType>
+          <additionalFieldNameAliases>
+            <element>country</element>
+          </additionalFieldNameAliases>
+          <isMultiValued v="true" />
         </indexRecordFieldMapping>
         <indexRecordFieldMapping>
-          <name>STANDARD_ZIP.value</name>
-          <indexFieldType>TEXT_NO_STOPWORDS</indexFieldType>
+          <name>IndexZIP.value</name>
+          <indexFieldType>ADDRESS</indexFieldType>
+          <additionalFieldNameAliases>
+            <element>zip</element>
+          </additionalFieldNameAliases>
+          <isMultiValued v="true" />
         </indexRecordFieldMapping>
       </indexFieldMappings>
     </indexRecordContainer>

entity/Person_entity/entityfields/indexphoneticfirstname/valueProcess.js

+import("system.result");
+import("system.vars");
+
+result.string(vars.get("$field.FIRSTNAME"));
\ No newline at end of file

entity/Person_entity/entityfields/indexphoneticlastname/valueProcess.js

+import("system.result");
+import("system.vars");
+
+result.string(vars.get("$field.LASTNAME"));
\ No newline at end of file

entity/Person_entity/recordcontainers/index/query.js

@@ -26,8 +26,10 @@ sqlQuery = 'select CONTACT.CONTACTID as "_uid_" '
     + ", PERSON.PERSONID "
     + ", PERSON.FIRSTNAME "
     + ", PERSON.LASTNAME "
+    + ", PERSON.FIRSTNAME as IndexPhoneticFirstname "
+    + ", PERSON.LASTNAME as IndexPhoneticLastname "
     + ", PERSON.SALUTATION "
-    + ", PERSON.GENDER "
+    + ", TRIM(PERSON.GENDER) as GENDER "
     + ', PERSON.TITLE as "TITLE.value"'
     + ", CONTACT.CONTACTID "
     + ", CONTACT.ORGANISATION_ID "

neonView/DuplicateScannerEdit_view/DuplicateScannerEdit_view.aod

@@ -23,10 +23,6 @@
           <name>c021711c-9f3e-454d-964c-1339e3463329</name>
           <entityField>ENTITY_TO_SCAN_NAME</entityField>
         </entityFieldLink>
-        <entityFieldLink>
-          <name>bc526338-19bb-4587-8321-7bd53c267c6d</name>
-          <entityField>BLOCK_SIZE</entityField>
-        </entityFieldLink>
       </fields>
     </genericViewTemplate>
     <neonViewReference>

preferences/_____PREFERENCES_PROJECT/_____PREFERENCES_PROJECT.aod

@@ -106,5 +106,9 @@
       <name>nominatim.pw</name>
       <property>useradito</property>
     </customStringProperty>
+    <customIntegerProperty>
+      <name>duplicates.dataBlockSize</name>
+      <property v="5000" />
+    </customIntegerProperty>
   </customProperties>
 </preferences>

process/RebuildAllDuplicateCaches_serverProcess/process.js

@@ -23,7 +23,7 @@ import("DuplicateScanner_lib");
 
 var filterName = "PersonDuplicates";
 var targetEntity = "Person_entity";
-var recordBlockSize = 5;
+var recordBlockSize = DuplicateScannerUtils.GetBlockSize();
 
 DuplicateScannerUtils.DeleteDuplicateClustersByTargetEntity(targetEntity);
 

process/RebuildDuplicatesCache_serverProcess/process.js

@@ -27,7 +27,7 @@ var filterName = vars.get("$local.filterName");
 var targetEntity = vars.get("$local.targetEntity");
 logging.log("filterName -> " + filterName);
 logging.log("targetEntity -> " + targetEntity);
-var recordBlockSize = DuplicateScannerUtils.GetBlockSizeForScanner(filterName, targetEntity);
+var recordBlockSize = DuplicateScannerUtils.GetBlockSize();
 let duplicateFieldsConfig = DuplicateScannerUtils.LoadIndexFieldsConfiguration(filterName, targetEntity);
 let resultFields = DuplicateScannerUtils.LoadResultFields(filterName, targetEntity);