Skip to content
Snippets Groups Projects
Commit 352c3365 authored by Sebastian Listl's avatar Sebastian Listl :speech_balloon:
Browse files

Merge branch 'cm-sp-1080178-AddressValidation' into '2021.1.0' 

Cm sp 1080178 address validation

See merge request xrm/basic!1018
parents 92143b8c 7f0e873a
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 135 additions and 111 deletions
preferences/_____PREFERENCES_PROJECT/_____PREFERENCES_PROJECT.aod
+ 15
− 10
View file @ 352c3365
......@@ -115,21 +115,12 @@
</customBooleanProperty>
<customStringProperty>
<name>nominatim.url</name>
<property>https://nom.adito.de/search.php</property>
<property>https://nom.adito.cloud/search.php</property>
</customStringProperty>
<customStringProperty>
<name>nominatim.user</name>
<property>aditouser</property>
</customStringProperty>
<customIntegerProperty>
<name>duplicates.dataBlockSize</name>
<property v="5000" />
</customIntegerProperty>
<customBooleanProperty>
<name>ai.salesprojectProbability</name>
<description></description>
<property v="false" />
</customBooleanProperty>
<customPasswordProperty>
<name>nominatim.pw</name>
<description></description>
......@@ -139,6 +130,15 @@
<name>nominatim.resultLimit</name>
<description>Amount of rows returned by the address search. Default is 10, maximum is 50.</description>
</customIntegerProperty>
<customJDitoProperty>
<name>nominatim.filterKeys</name>
<description></description>
<property>%aditoprj%/preferences/_____PREFERENCES_PROJECT/customproperties/nominatim.filterkeys/property.js</property>
</customJDitoProperty>
<customIntegerProperty>
<name>duplicates.dataBlockSize</name>
<property v="5000" />
</customIntegerProperty>
<customBooleanProperty>
<name>AD.active</name>
<description></description>
......@@ -187,6 +187,11 @@
<description>Default VAT in percent</description>
<property>19</property>
</customStringProperty>
<customBooleanProperty>
<name>ai.salesprojectProbability</name>
<description></description>
<property v="false" />
</customBooleanProperty>
<customStringProperty>
<name>bulkmail.fileStorage</name>
<description>Save Location for send Mails. Path is relative to Serverpath</description>
......
preferences/_____PREFERENCES_PROJECT/customproperties/nominatim.filterkeys/property.js 0 → 100644
+ 1
− 0
View file @ 352c3365
["building", "office", "townhall"]
\ No newline at end of file
process/Location_lib/process.js
+ 56
− 7
View file @ 352c3365
import("Sql_lib");
import("system.db");
import("system.logging");
import("Address_lib");
import("Util_lib");
......@@ -170,12 +172,12 @@ function LocationFinder ()
{
this._minAccuracy = AddressParts.ZIPCODE();
this._accuracyOrder = [
AddressParts.BUILDINGNO(),
AddressParts.ADDRESS(),
AddressParts.ZIPCODE(),
AddressParts.CITY(),
AddressParts.STATE(),
AddressParts.COUNTRY()
AddressParts.BUILDINGNO(),
AddressParts.ADDRESS(),
AddressParts.ZIPCODE(),
AddressParts.CITY(),
AddressParts.STATE(),
AddressParts.COUNTRY()
];
this._getLocationFn = LocationFinder._getLocationWithNominatim;
this._isServiceAvailable = WsValidationUtils.isWsEnabled("TYPE_FULLADDRESS_NOMINATIM");
......@@ -254,7 +256,7 @@ LocationFinder.prototype.isEnabled = function ()
* or the min accuracy has been exceeded (the default min accuracy is zipCode).
*
* @param {AddressObject} pAddress an object containing the address data
* @return {Object} object with the properties "lat", "lon" and "accuracy", or null if the address could not be found
* @return {Object} object with the properties "lat", "lon", "requestResult" and "accuracy", or null if the address could not be found
*/
LocationFinder.prototype.getGeoLocation = function (pAddress)
{
......@@ -279,6 +281,7 @@ LocationFinder.prototype.getGeoLocation = function (pAddress)
return {
lat: location.lat,
lon: location.lon,
requestResult: JSON.stringify(location),
accuracy: mostAccurateField
};
}
......@@ -286,4 +289,50 @@ LocationFinder.prototype.getGeoLocation = function (pAddress)
searchAddress[mostAccurateField] = null;
}
return null;
}
/**
* Update LAT, LON and VALIDATION of the given Addresses
* This function is call-by-reference and the errors of the updates will be returned by pMalformedAddressData.
*
* @param {pMalformedAddressData} to collect the errors of the updates
* @param {pAddresses} pAddress an object containing the address data
*
* @return {void}
*/
LocationFinder.PagingFunctionSetLocations = function (pMalformedAddressData, pAddresses)
{
var locationUpdates = [];
pAddresses.forEach(function ([addressId, address, buildingno, zip, city, country, state, validationResult])
{
var addressData = new AddressObject(address, buildingno, zip, city, country, state);
try
{
var location = locationSearch.getGeoLocation(addressData);
if (location)
{
let fields = ["LAT", "LON"];
let values = [location.lat, location.lon];
if (Utils.isNullOrEmpty(validationResult))
{
fields.push("VALIDATION_RESULT");
values.push(location.requestResult);
}
locationUpdates.push(["ADDRESS", fields, null, values,
newWhere("ADDRESS.ADDRESSID", addressId).buildCondition()]);
}
}
catch (pException)
{
pMalformedAddressData.push(addressId);
}
});
if (locationUpdates.length > 0)
{
db.updates(locationUpdates);
}
}
\ No newline at end of file
process/WsValidation_lib/process.js
+ 56
− 45
View file @ 352c3365
......@@ -78,58 +78,65 @@ WsValidationType.get = function(pKey)
}
}
function _processNominatimAddressLookup(pWsResult, pValue)
{
if (pWsResult != null)
function _processNominatimAddressLookup(pWsResult, pValue)
{
return pWsResult.map(function(pPlaceData)
let filterKeys = JSON.parse(project.getPreferenceValue("custom.nominatim.filterKeys", "[]"));
let filteredResults = [];
if (pWsResult != null)
{
if (pPlaceData.address == undefined)
pPlaceData.address = {};
pWsResult.forEach(function(pPlaceData)
{
if (Utils.isEmpty(filterKeys) || filterKeys.indexOf(pPlaceData.osm_type) > -1 || filterKeys.indexOf(pPlaceData.type) > -1 || filterKeys.indexOf(pPlaceData["class"]) > -1)
{
if (pPlaceData.address == undefined)
pPlaceData.address = {};
var city = pPlaceData.address.town
|| pPlaceData.address.city
|| pPlaceData.address.village
|| pPlaceData.address.hamlet;
var city = pPlaceData.address.town
|| pPlaceData.address.city
|| pPlaceData.address.village
|| pPlaceData.address.hamlet;
var cityext = pPlaceData.address.suburb
|| pPlaceData.address.city_district;
var cityext = pPlaceData.address.suburb
|| pPlaceData.address.city_district;
var road = pPlaceData.address.road
|| pPlaceData.address.street
|| pPlaceData.address.path
|| pPlaceData.address.address27;
var road = pPlaceData.address.road
|| pPlaceData.address.street
|| pPlaceData.address.path
|| pPlaceData.address.address27;
return {
placeId: pPlaceData.place_id,
value: pValue,
displayName: pPlaceData.display_name,
zipCode: pPlaceData.address.postcode,
city: city,
cityExt: cityext,
country: pPlaceData.address.country_code.toUpperCase(), //4
state: pPlaceData.address.state,
road: road, //8
buildingNo: pPlaceData.address.house_number,
lat: pPlaceData.lat,
lon: pPlaceData.lon
};
});
}
filteredResults.push({
placeId: pPlaceData.place_id,
value: pValue,
displayName: pPlaceData.display_name,
zipCode: pPlaceData.address.postcode,
city: city,
cityExt: cityext,
country: pPlaceData.address.country_code.toUpperCase(), //4
state: pPlaceData.address.state,
road: road, //8
buildingNo: pPlaceData.address.house_number,
lat: pPlaceData.lat,
lon: pPlaceData.lon
});
}
});
}
return [];
}
return filteredResults;
}
function _processNominatimAddressLocation (pWsResult)
{
if (pWsResult == null || pWsResult.length === 0)
return null;
function _processNominatimAddressLocation (pWsResult, pValue)
{
pWsResult = _processNominatimAddressLookup(pWsResult, pValue);
if (Utils.isNullOrEmpty(pWsResult))
{
return null;
}
return {
lat: pWsResult[0].lat,
lon: pWsResult[0].lon
};
}
return pWsResult[0];
}
/**
* request using a custom webservice
......@@ -278,6 +285,7 @@ WsValidationType.get = function(pKey)
if (!Utils.isNullOrEmpty(pParameters))
{
var nominatimRequest = new NominatimRequest(url)
.includeAddressDetails()
.setResultLimit(1)
.setParameters(pParameters);
......@@ -361,7 +369,8 @@ WsValidationUtils.setAddressFields = function(pJSON)
//"$field.DISTRICT": data.district, not needed currently
"$field.STATE": data.state,
"$field.ADDRESS": data.road,
"$field.BUILDINGNO": data.buildingNo
"$field.BUILDINGNO": data.buildingNo,
"$field.VALIDATION_RESULT": JSON.stringify(data)
}
for (let field in toSet)
......@@ -383,8 +392,10 @@ WsValidationUtils.setAddressFields = function(pJSON)
WsValidationUtils.valueFromJSON = function(pJSON)
{
var data = WsValidationUtils.parseJSONArray(pJSON);
if (data == null)
if (Utils.isNullOrEmpty(data) || Utils.isNullOrEmpty(data.value))
{
return null;
}
return data.value;
}
......
process/setMissingAddressLocations_serverProcess/process.js
+ 3
− 24
View file @ 352c3365
......@@ -10,34 +10,13 @@ var malformedAddressData = [];
if (locationSearch.isEnabled())
{
newSelect("ADDRESSID, ADDRESS, BUILDINGNO, ZIP, CITY, COUNTRY, PROVINCE")
newSelect("ADDRESSID, ADDRESS, BUILDINGNO, ZIP, CITY, COUNTRY, PROVINCE, VALIDATION_RESULT")
.from("ADDRESS")
.where("ADDRESS.LAT is null")
.or("ADDRESS.LON is null")
.pageSize(400)
.forEachPage(function (addresses)
{
var locationUpdates = [];
addresses.forEach(function ([addressId, address, buildingno, zip, city, country, state])
{
var addressData = new AddressObject(address, buildingno, zip, city, country, state);
try
{
var location = locationSearch.getGeoLocation(addressData);
if (location)
{
locationUpdates.push(["ADDRESS", ["LAT", "LON"], null, [location.lat, location.lon],
newWhere("ADDRESS.ADDRESSID", addressId).buildCondition()]);
}
}
catch (pException)
{
malformedAddressData.push(addressId);
}
});
if (locationUpdates.length > 0)
db.updates(locationUpdates);
.forEachPage(function (pAddresses) {
LocationFinder.PagingFunctionSetLocations(malformedAddressData, pAddresses);
});
if (Utils.isNotNullOrEmptyString(malformedAddressData))
{
......
process/updateAllAddressLocations_serverProcess/process.js
+ 4
− 25
View file @ 352c3365
......@@ -10,34 +10,13 @@ var malformedAddressData = [];
if (locationSearch.isEnabled())
{
newSelect("ADDRESSID, ADDRESS, BUILDINGNO, ZIP, CITY, COUNTRY, PROVINCE")
newSelect("ADDRESSID, ADDRESS, BUILDINGNO, ZIP, CITY, COUNTRY, PROVINCE, VALIDATION_RESULT")
.from("ADDRESS")
.pageSize(400)
.forEachPage(function (addresses)
{
var locationUpdates = [];
addresses.forEach(function ([addressId, address, buildingno, zip, city, country, state])
{
var addressData = new AddressObject(address, buildingno, zip, city, country, state);
try
{
var location = locationSearch.getGeoLocation(addressData);
if (location)
{
locationUpdates.push(["ADDRESS", ["LAT", "LON"], null, [location.lat, location.lon],
newWhere("ADDRESS.ADDRESSID", addressId).buildCondition()]);
}
}
catch (pException)
{
malformedAddressData.push(addressId);
}
});
if (locationUpdates.length > 0)
db.updates(locationUpdates);
.forEachPage(function (pAddresses) {
LocationFinder.PagingFunctionSetLocations(malformedAddressData, pAddresses);
});
if (Utils.isNotNullOrEmptyString(malformedAddressData))
if (!Utils.isNullOrEmpty(malformedAddressData))
{
errMsg = "updateAllAddressLocations_serverProcess: Something went wrong, during updating address locations, with the following uids: ";
malformedAddressData.forEach(function(pMalformedData)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment