process.js

import("system.logging");
import("system.notification");
import("system.translate");
import("system.datetime");
import("KeywordRegistry_basic");
import("system.eMath");
import("Leadimport_lib");
import("system.text");
import("system.util");
import("Sql_lib");
import("system.db");
import("system.vars");

var importdefid = vars.get("$local.importdefid");
var assignmentRowId = vars.get("$local.assignmentRowId");
var binId = vars.get("$local.binId"); 
var recordSep = vars.get("$local.recordSep"); 
var fieldSep = vars.get("$local.fieldSep");
var fieldLimit = vars.get("$local.fieldLimit");  
var source = vars.get("local.source");
var user = vars.get("$local.user");
var currDate = vars.get("$local.currDate");    
var userId = vars.get("$local.userId");    
var importName = vars.get("$local.importName");    
var posanz;
var allContactData = {};
var errorCount = 0;
var exeptions = "";

var data = db.getBinaryContent(binId, SqlUtils.getBinariesAlias());
data = util.decodeBase64String(data);

data = text.parseCSV(data.replace(/(^\s+)|(\s+$)/g,""), recordSep, fieldSep, fieldLimit.charAt(0));//get data from file

var ImportDate =  vars.getString("$sys.date");
var ImportFieldDef = LeadImportUtils.getImportFieldDef(importdefid); // [Fieldnumber, Mappingfield]

var DataFieldCount = data[0].length;
//  Attributes that should be created for Organisations and persons
var AttrObject = LeadImportUtils.getLeadAttr(importdefid);

var FieldDef = LeadImportUtils.addArray(ImportFieldDef, new Array(DataFieldCount++, "SOURCE", ""));

FieldDef.push(new Array(DataFieldCount++, "DATE_NEW", ""));

//  für Tabellen Felder und Typen ermitteln
var DataTables = new Array("ORGANISATION", "PERSON", "CONTACT", "ADDRESS", "LEAD", "COMMUNICATION", "AB_ATTRIBUTERELATION");
var DataFields = LeadImportUtils.getDataFields(DataTables);
var DataTypes = LeadImportUtils.getDataTypes(DataFields, DataTables);

// Check if a BUILDINGNO is declared
var addressPos = getFieldPos(ImportFieldDef, "ADDRESS");
var noBuildingNr = (getFieldPos(ImportFieldDef, "BUILDINGNO").length == 0 && addressPos.length > 0);
if (noBuildingNr)
    FieldDef.push(new Array(DataFieldCount++, "BUILDINGNO", ""));

// Check if a COUNTRY is declared
var noCountry = (getFieldPos(ImportFieldDef, "COUNTRY").length ==  0); 
if (noCountry)
    FieldDef.push(new Array(DataFieldCount++, "COUNTRY", ""));

// Check if a ORGNAME is declared frequently
var posOrgName = getFieldPos(ImportFieldDef, "NAME"); 

//// Check if a ORGINFO is declared frequently
var posOrginfo = getFieldPos(ImportFieldDef, "INFO");

// Data without headline
var dsanz = 1;

// set the mappings
LeadImportUtils.mapping = LeadImportUtils.getMapping(importdefid);


try
{
    for (; dsanz < data.length; dsanz++)
    {
        //iterate over the fielddefinitions and set the values
        var impvalues = LeadImportUtils.addArray(data[dsanz], source);	
        impvalues.push(ImportDate);
        
        //  if there is no BUILDINGNO the select it from the ADDRESS
        if (noBuildingNr)
        {
            var pos =  addressPos[0];
            var buildingNr = "";
            var address = impvalues[pos];

            if (address != "")
            {
                // split ADDRESS and BUILDINGNO
                var arr = address.match(/^[^0-9]+|[0-9]+.*$/g);
                
                //Regexp can throw an error - then the array is empty
                if (arr && arr[0])
                {
                    impvalues[pos] = arr[0].replace(/(^\s+)|(\s+$)/g,"");
                    if (arr[1]) buildingNr = arr[1];
                }
            }
            impvalues.push(buildingNr);
        }
        // if there is no COUNTRY then COUNTRY = DE
        if (noCountry) impvalues.push("DE");
				
        // if a ORGNAME is declared frequently
        if (posOrgName.length > 1)
        {					
            for (posanz = 1; posanz < posOrgName.length; posanz++)
            {
                if (impvalues[posOrgName[posanz]] != "")
                    impvalues[posOrgName[0]] += "\n" +  impvalues[posOrgName[posanz]]; //impvalues[ posOrgName[ posanz ]] ist das Import-Datum
            }
        }
        //  if a ORGINFO is declared frequently
        if (posOrginfo.length > 1)
        {
            for (posanz = 1; posanz < posOrginfo.length; posanz++)
            {
                if (impvalues[posOrginfo[posanz]] != "" )
                    impvalues[posOrginfo[0]] += "\n" +  impvalues[posOrginfo[posanz]]; 
            }
        }
        //get the position of the COUNTRY
        var countrypos = getFieldPos (FieldDef, "COUNTRY")[0];
        
        //check the COUNTRY
        impvalues[countrypos] = LeadImportUtils.checkCountry(impvalues[countrypos]);
        
        if (!(getFieldPos(ImportFieldDef, "ISOLANGUAGE").length ==  0)){
            //get the position of the ISOLANGUAGE
            var isolanguepos = getFieldPos (FieldDef, "ISOLANGUAGE")[0];
        
            //mapp and check the ISOLANGUAGE
            impvalues[isolanguepos] = LeadImportUtils.getMappedOutputvalue("ISOLANGUAGE", impvalues[isolanguepos]);
            impvalues[isolanguepos] = LeadImportUtils.checkISOLanguage(impvalues[isolanguepos]);
        }
        
        try//import data
        {
            var dupCheck = LeadImportUtils.importData(DataFields, DataTypes, FieldDef, impvalues, importdefid, AttrObject, source, user, currDate);

            allContactData[dupCheck[0]] = [dupCheck[1], dupCheck[2]];   //persObj, orgObj
        }
        catch(ex)//if error increase the errorCount and write a rhinoException into the string for the error file
        {
            var exeption = "Zeile " + dsanz + ": " + logging.toLogString(ex.rhinoException != undefined ? ex.rhinoException : ex, true) + "\r\n";
            exeptions = exeptions + exeption;
            errorCount++;
            logging.log(exeption);
        }
    }
}
catch(err)
{
    errorCount++;
    logging.log(err);
}

try 
{
    LeadImportUtils.scanLeadDups(allContactData);//search for duplicates
}
catch (err)
{
    logging.log(err);
}


var rows = eMath.subInt(dsanz, 1);
rows = eMath.subInt(rows, errorCount);//datasetcount - errors = inserts

var updTable = "LEADIMPORT";
var updFields =["IMPORT_DATE", "DATE_EDIT", "USER_EDIT", "STATUS"];
var updVals = [ImportDate, currDate, user, $KeywordRegistry.importStatus$transfered()];

var logTable = "LEADLOG";
var logFields =["LEADLOGID", "ROWSINSERTED", "IMPORTERRORS", "LEADIMPORT_ID", "IMPORTSOURCE", "USER_NEW", "DATE_NEW"];
var logVals = [util.getNewUUID(), rows, errorCount, importdefid, source, user, currDate];

db.insertData(logTable, logFields, null, logVals);
newWhere("LEADIMPORT.LEADIMPORTID", importdefid)
    .updateData(true, updTable, updFields, null, updVals);
newWhere("LEADTEMP.ROW_ID", importdefid)
    .deleteData(true, "LEADTEMP");//delete existing temp data

if(exeptions != "")//insert error document
    db.insertBinary("LEADLOG", "ERRORLOG", logVals[0], 
        "", util.encodeBase64String(exeptions, "UTF-8"), "Error-Log " + source + " " + datetime.toDate(currDate, "dd.MM.yyyy HH:mm") , "", "", SqlUtils.getBinariesAlias());
        
if (userId)// if there is an user - show a notification to this user when the import is complete
{
    var description = translate.withArguments("%0 leads imported, %1 errors", [rows, errorCount]);
    notification.addNotification(util.getNewUUID(), null, null, null, "LeadImport_Notification", null, notification.PRIO_NORMAL, 2, notification.STATE_UNSEEN, [userId], importName, description);
}


// Returns the positions of the field name 
function getFieldPos (pFieldDef, pFieldName)
{
    var multi = false;
    var pos = new Array();
    for (let i = 0; i < pFieldDef.length; i++)
    {
        if (pFieldDef[i][1] == pFieldName)
        {
            pos.push(pFieldDef[i][0]);
            if (multi) pFieldDef[i][1] = "NULL";
            multi = true;
        }
    }
    return pos;
}