Something went wrong on our end
-
Johannes Hörmann authoredJohannes Hörmann authored
process.js 69.94 KiB
import("system.db");
import("system.vars");
import("system.translate");
import("system.logging");
import("system.datetime");
import("system.fileIO");
import("system.swing");
import("system.util");
import("system.text");
import("ImporterCustomMappingFunctions_lib");
import("ImporterMappingFunctions_lib");
/*
┌─────────────────────────────────────────────────────────────┐
│ importer module constructor function │
│ DO NOT TOUCH - use lib_importerCustomMappingFunctions │
└─────────────────────────────────────────────────────────────┘
Frequently asked questions:
──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
Q: Why does the importer have a own versionsystem, if there exists git?
A: Because its a standalone module and there is no modularisation-system right now.
So you may want to update the IMPORTER-module and have to keep an eye on the verison.
Q: Where can I see the version of the importer?
A: There is a @version-tag in the documentation of the Importer-function (constructor-function for the object)
*/
/*
* the importer class, main object
* @param {String []} pConfig req ( Name, FunktionsArt, Details )
*
* @version 2.1
*
*
* @return {void}
*/
function Importer(pConfig)
{
var batchNum = 0;
var batchStart = 0;
var batchList;
var mappingtimer = [];
var exectimer = {
insertData: 0,
updateData: 0,
outbuffer: 0,
logging: 0,
getdata: 0,
updateDecisionTime: 0,
insertUpdateActions: 0,
dbDataModifiedTime: 0,
dbInsertTime: 0,
dbUpdateTime: 0,
dataLoopTime: 0,
mapPerRecordTime: 0,
outputPerRecordTime: 0
};
var startTime = datetime.date();//this will be overwriten when starting the importing-process
var stopTime;
var logBuffer = [];
this.insertArray = [];
this.updateArray = [];
this.LogLevels = {
Minimal: 0,
Error : 1,
Warning : 2, Info : 3,
Debug : 4,
Preview : 5
};
this.Cases = {
Lower: 0,
Upper: 1,
Unchanged: 2
};
this.Process = "";
this.Config = pConfig;
this.BatchSize = 50;
this.MaxRows = 0;
this.Preview = false;
this.Debug = false;
this.Log = "CONSOLE";
this.fileInputCharset = "UTF8";
this.LogLevel = this.LogLevels.Warning;
this.enableLogBuffer = false;
this.InputRecord = new Array();
this.OutputRecord = new Object();
this.ImportUser = "IMPORTER";
this.FuncBuffer = new Object();
this.CompleteUpdate = true;
this.ErrorLog = "";
this.DataType = null;
this.KeyColumn = new Object(); // contains key info as in KeyColumn["tbl.col"] = "I" | "U" | "I+U"
this.UseAOType = true; //default true, so that AOType will be uses (for older systems)
this.UseUUID = true; //default false, so that db.getNewID() instead of util.getNewUUID() would be uses
this.TableCase = this.Cases.Upper; //default is uppercase
this.ColumnCase = this.Cases.Upper; //default is uppercase
this.skipEmptyValue = true;
this.AttributeCache = undefined;
this.useAttributeCache = false;
this.attributeCacheLoadedSuccessfully = false;
this.loadAttributeCache = function(){
this.writeLog(this.LogLevels.Info, "[LoadAttributeCache]loading Attibute cache");
try
{
this.AttributeCache = {};
let TMP = 0;
var cD = {
attrNameL1: TMP
,attrNameL2: ++TMP
,attrNameL3: ++TMP
,attrIdL1: ++TMP
,attrIdL2: ++TMP
,attrIdL3: ++TMP
,attrObjectL1: ++TMP
,attrObjectL2: ++TMP
,attrCompL1: ++TMP
,attrCompL2: ++TMP
,attrSqlIdFieldL1: ++TMP
,attrSqlDisplayFieldL1: ++TMP
,attrSqlFromFieldL1: ++TMP
,attrSqlWhereFieldL1: ++TMP
,attrSqlIdFieldL2: ++TMP
,attrSqlDisplayFieldL2: ++TMP
,attrSqlFromFieldL2: ++TMP
,attrSqlWhereFieldL2: ++TMP
};
var sqlStr = "select "
+ this.getColumnCase( "A.ATTRNAME, B.ATTRNAME, C.ATTRNAME "
+ ",A.ATTRID, B.ATTRID, C.ATTRID "
+ ",AO_A.ATTROBJECT, AO_B.ATTROBJECT "
+ ",A.ATTRCOMPONENT, B.ATTRCOMPONENT " + ",A.SQLIDCOLUMN, A.SQLSHOWCOLUMN, A.SQLFROMDEF, A.SQLWHERE "
+ ",B.SQLIDCOLUMN, B.SQLSHOWCOLUMN, B.SQLFROMDEF, B.SQLWHERE "
+ "")
+ " from " + this.getTableCase("ATTR A")
+ " left join " + this.getTableCase("ATTR B") + " on " + this.getColumnCase("A.ATTRID") + " = " + this.getColumnCase("B.ATTR_ID")
+ " left join " + this.getTableCase("ATTR C") + " on " + this.getColumnCase("B.ATTRID") + " = " + this.getColumnCase("C.ATTR_ID")
+ " left join " + this.getTableCase("ATTROBJECT AO_A") + " on " + this.getColumnCase("AO_A.ATTR_ID") + " = " + this.getColumnCase("A.ATTRID")
+ " left join " + this.getTableCase("ATTROBJECT AO_B") + " on " + this.getColumnCase("AO_B.ATTR_ID") + " = " + this.getColumnCase("B.ATTRID")
+ " where " + this.getColumnCase("A.ATTR_ID") + " is null "//if it has an attr_id it's level 2 an with that ATTR B (or ATTR C)
+ "";
this.writeLog(this.LogLevels.Info, "[LoadAttributeCache]executed select is: " + sqlStr);
var data = db.table(sqlStr, this.Config.AliasTo);
var attributeTree = {};
for(var i = 0, j = data.length; i < j; i++)
{
let row = data[i];
if (attributeTree[row[cD.attrNameL1]] == undefined)
attributeTree[row[cD.attrNameL1]] = {
id: row[cD.attrIdL1]
,component: row[cD.attrCompL1]
,usedInFrames: {}
,children: {}
};
if (row[cD.attrObjectL1] != "")
attributeTree[row[cD.attrNameL1]].usedInFrames[row[cD.attrObjectL1]] = true;
else
this.writeLog(this.LogLevels.Warning, '[loadAttributeCache]attribute "' + row[cD.attrNameL1] + '" has no ATTROBJECT');
//selectCombo, init for hybrid-cache (loaded once, when attribute is really needed):
if (row[cD.attrCompL1] == "7"
&& attributeTree[row[cD.attrNameL1]].loadedSelectCombo == undefined)
{
attributeTree[row[cD.attrNameL1]].loadedSelectCombo = false;
attributeTree[row[cD.attrNameL1]].selectComboSql = "select " + row[cD.attrSqlIdFieldL1]
+ " ," + row[cD.attrSqlDisplayFieldL1]
+ " from " + row[cD.attrSqlFromFieldL1]
+ (row[cD.attrSqlWhereFieldL1] != "" ? " where " : "") + row[cD.attrSqlWhereFieldL1];
}
if (row[cD.attrNameL2] != "")
{
if (attributeTree[row[cD.attrNameL1]].children[row[cD.attrNameL2]] == undefined)
attributeTree[row[cD.attrNameL1]].children[row[cD.attrNameL2]] = {
id: row[cD.attrIdL2]
,component: row[cD.attrCompL2]
,usedInFrames: {}
,children: {}
};
if (row[cD.attrObjectL2] != "")
attributeTree[row[cD.attrNameL1]].children[row[cD.attrNameL2]].usedInFrames[row[cD.attrObjectL2]] = true;
else
this.writeLog(this.LogLevels.Warning, '[loadAttributeCache]attribute "' + row[cD.attrNameL1] + "|" + row[cD.attrNameL2] + '" has no ATTROBJECT');
//selectCombo, init for hybrid-cache:
if (row[cD.attrCompL2] == "7"
&& attributeTree[row[cD.attrNameL1]].children[row[cD.attrNameL2]].loadedSelectCombo == undefined)
{
attributeTree[row[cD.attrNameL1]].children[row[cD.attrNameL2]].loadedSelectCombo = false;
attributeTree[row[cD.attrNameL1]].children[row[cD.attrNameL2]].selectComboSql = "select " + row[cD.attrSqlIdFieldL2]
+ " ," + row[cD.attrSqlDisplayFieldL2]
+ " from " + row[cD.attrSqlFromFieldL2]
+ (row[cD.attrSqlWhereFieldL2] != "" ? " where " : "") + row[cD.attrSqlWhereFieldL2];
}
//this nomally happens only if component is combo
if (row[cD.attrNameL3] != "")
{
if (attributeTree[row[cD.attrNameL1]].children[row[cD.attrNameL2]].children[row[cD.attrNameL3]] == undefined) attributeTree[row[cD.attrNameL1]].children[row[cD.attrNameL2]].children[row[cD.attrNameL3]] = {
id: row[cD.attrIdL3]
};
}
}
// //jetzt muss geprüft werden ob sich die ATTRIDs unterscheiden, wenn ja bedeutet dass, das ein Attributname mehrfach vergeben wurde
// //dann darf keiner der Daten in den Cache geladen werden, dass würde nur zu fehlerhaften daten führen
// var existingAttrid = this.AttributeCache[ attrname ].ATTRID;
// if( existingAttrid != attrid )
// {
// this.AttributeCache[ attrname ] = null; //cache sicherheitshalber erst freigeben
// this.AttributeCache[ attrname ] = undefined;
// this.writeLog(this.LogLevels.Error, "[LoadAttributeCache] Multiple Attributes were found for attribute: '" + attrname + "'."
// + "\r\nNo Attribute with attrname '" + attrname + "' will be loaded into the cache. ");
// }
}
this.AttributeCache = attributeTree;
this.attributeCacheLoadedSuccessfully = true;
}
catch(ex)
{
this.attributeCacheLoadedSuccessfully = false;
this.writeLog(this.LogLevels.Error, "[LoadAttributeCache]Error while loading Attributes:\r\n" + logging.toLogString(ex) );
}
if( this.attributeCacheLoadedSuccessfully )
this.writeLog(this.LogLevels.Info, "[LoadAttributeCache]Loaded Attibute cache successfully");
else
this.writeLog(this.LogLevels.Info, "[LoadAttributeCache]Failed loading Attibute cache");
}
this.recordCounts = {//is initialized with numbers in the this.process-function
total: null
,skip: null
,update: null
,insert: null
,unchanged: null
};
Object.defineProperty(this.recordCounts, 'unchanged', {
//"this" is here the passed object
get: function (){ return (this.total - this.insert - this.update - this.skip)}
,set: function (){throw new Error("importers recordCounts cannot be set; it is read-only")}
});
this.process = function()
{
this.mappingLength = this.Config.Mapping.length;
var i;
var t;
var pkid;
var c;
var cond;
var s;
var pImporter = this;
this.recordCounts.total = 0;
this.recordCounts.skip = 0;
this.recordCounts.insert = 0;
this.recordCounts.update = 0;
// Infoobject for Callbackfunction
var recordStack = {
sourceDataSet: [],
insertUpdate: "",
insertData: {
successful: true,
array: []
},
updateData: {
successful: true, array: []
},
exception: [],
pId: ""
};
var daten;
var skip;
var headercount = 0;
var batchlist = [];
var idcolumn;
var importtype = "S"; // F,S or M for "file", "single" or "multi" batch, default to single batch
batchStart = 0; // current batch start index
var logstring = ""; // contains the log file, if not writing to server log
var requiredTargets = new Object(); // stores the required target columns
// prepare logging stuff
if(this.Log != "" && this.Log != undefined && this.Log.split(".")[0] == "$global") vars.set(this.Log, "");
if(this.Preview == true) this.LogLevel = this.LogLevels.Preview;
if (this.enableLogBuffer)
logBuffer = [];
else
logBuffer = ["logBuffer is not enabled; you cannot use getLogMessages(...)"];
startTime = datetime.date();
s = datetime.toDate(startTime, translate.text("yyyy-MM-dd HH:mm:ss"), "UTC");
this.writeLog(this.LogLevels.Minimal, "Started at " + s + " (UTC)");
logging.log("[IMPORT] Start at " + s + " UTC");
if(this.Preview) this.writeLog(this.LogLevels.Info, "======== import running in preview mode ========");
if (this.useAttributeCache)
this.loadAttributeCache();
this.writeLog(this.LogLevels.Debug, "Building mapping tables.");
// get all column type from all tables of the target alias
this.DataType = this.getDataTypes(this.Config.AliasTo);
// build an object with the data types for each column and
// check for key fields and collect them in parmObject.KeyRecord
for(i=0; i < this.mappingLength; i++)
{
mappingtimer[i] = 0; // init mapping timer
var target = this.Config.Mapping[i][1]["Target"];
if(target != undefined && target.substr(0,4).toLowerCase() != "var.") // exclude var storage
{
// collect a required target, if we find one (these columns must not be empty)
requiredTargets[target] = (this.Config.Mapping[i][1]["Required"] == true);
// collect all key columns in an object array
// KeyColumn[table ][column] = action1;action2...
var coldata = target.split(".");
if(this.Config.Mapping[i][1]["Key"] != undefined) //2009-05-13 TR changed, because on insertcommand "insert" or "update" there can be no Action
{
if(this.KeyColumn[ this.getTableCase(coldata[0]) ] == undefined) this.KeyColumn[ this.getTableCase(coldata[0]) ] = new Object();
if(this.KeyColumn[ this.getTableCase(coldata[0]) ][ this.getColumnCase(coldata[1]) ] == undefined) this.KeyColumn[ this.getTableCase(coldata[0]) ][ this.getColumnCase(coldata[1]) ] = "key";
this.setDefaultAction(this.Config.Mapping[i][1]);
this.KeyColumn[ this.getTableCase(coldata[0]) ][ this.getColumnCase(coldata[1]) ] = this.KeyColumn[ this.getTableCase(coldata[0]) ][ this.getColumnCase(coldata[1]) ] + ";" + this.Config.Mapping[i][1]["Action"];
}
}
}
batchNum = 0;
headercount = this.Config.HeaderLines;
if(headercount == undefined) headercount = 0;
this.writeLog(this.LogLevels.Warning, "Beginning data import.");
// main loop
while(daten = this.getNextBatch(this.Config), daten != null )
{
var dataLoopTimeStart = datetime.date();
this.writeLog(this.LogLevels.Info, "Processing batch #" + batchNum);
// import each row of the current batch
for(var bi=0; bi < daten.length; bi++)
{
recordStack.exception ="";
recordStack.sourceDataSet = daten[bi];
recordStack.insertData = {
successful: true,
array: []
}
recordStack.updateData = {
successful: true,
array: []
}
// skip empty data rows
if(daten[bi].length == 1 && daten[bi][0] == "")
{
if(this.Debug == true) {
this.writeLog(this.LogLevels.Debug, "Skipping empty row " + bi);
}
continue;
}
this.recordCounts.total++;
skip = false;
// skip header rows (usually file import only)
if(bi < headercount)
{
if(this.Preview == true && this.Debug == true)
{
this.writeLog(this.LogLevels.Debug, "Skipping header row " + bi);
}
continue;
}
if (this.Debug)
this.writeLog(this.LogLevels.Debug, "Row " + bi);
this.InputRecord = daten[bi]; // assign input fields for this record
this.OutputRecord = new Object(); // clear output fields for this record
var mapRecordMapTimeStart = datetime.date();
// call mapping functions
for(i=0, ml = this.mappingLength; i < ml; i++)
{
try
{
var fname = this.Config.Mapping[i][0].name;//name is inherited function-property
if (this.Debug)
this.writeLog(this.LogLevels.Debug, "Calling mapping function " + fname);
// call mapping function and time it
t = datetime.date();
var mapresult = this.Config.Mapping[i][0].call(this, this.Config.Mapping[i][1]);
t = datetime.date() - t;
mappingtimer[i] += t;
// if the mapping did not succeed, log function and ID value
if(mapresult == false)
{
this.writeLog(this.LogLevels.Error, "Mapping function " + fname + " failed for row " + this.recordCounts.total);
recordStack.exception = "Fehler Zeile: " + this.recordCounts.total + " - Info: Mapping function " + fname + " failed for row " + this.recordCounts.total + " Column: " + this.Config.Mapping[i][1]["Target"];
skip = true;
}
}
catch(ex)
{
logging.log(ex["rhinoException"] != undefined ? ex["rhinoException"] : ex)
this.writeLog(this.LogLevels.Error, "Exception in mapping function [" + fname + "] for input row " + this.recordCounts.total+ " - " + this.Config.Mapping[i][1]["Target"]);
recordStack.exception = logging.toLogString(ex["rhinoException"] != undefined ? ex["rhinoException"] : ex, true) + " - Column: " + this.Config.Mapping[i][1]["Target"];
skip = true;
}
}
exectimer.mapPerRecordTime += datetime.date() - mapRecordMapTimeStart;
if(skip == false) // do we continue with this record?
{
// if a debug callback function has been defined, call it
if(this.DebugCallback != null && typeof(this.DebugCallback) == "function")
{
this.DebugCallback(this.OutputRecord);
}
var tt = datetime.date();
var outdata = new Object();
for(var tbl in this.OutputRecord) // loop thru all output tables
{
for(var col in this.OutputRecord[ tbl ]) // loop thru all output columns
{
if(tbl.toLowerCase() != "var") // exclude var storage
{
for(var action in this.OutputRecord[tbl][col])
{
if(outdata[tbl] == undefined) outdata[tbl] = "";
outdata[tbl] = outdata[tbl] + ";" + action;
// check for required data mappings and set skip again, we check this later
skip = (requiredTargets[tbl + "." + col] == true && (this.OutputRecord[tbl][col][action] == undefined || this.OutputRecord[tbl][col][action] == ""));
}
}
}
}
tt = datetime.date() - tt;
exectimer["outbuffer"] += tt;
var outputRecordTimeStart = datetime.date();
// loop thru all distinct tables, maybe we have more than one record to write
for(t in this.OutputRecord)
{
var spalten = [];
var typen = [];
var werte = [];
// we do insert-for-new and update-for-existing
if(skip == false && this.Config.ImportCommand == "insert+update")
{
var insertUpdateStart = datetime.date();
pkid = "0"; // assume we do not find an existing record
// if no key values have been specified, there is no need to check
// so we do an insert
if(this.KeyColumn[t] != undefined )
{
// check for existing data row based on key values
cond = this.generateKeyCondition(t);
var sql = "select count(*) from " + t + " where " + cond;
this.writeLog(this.LogLevels.Debug, "Insert/Update decision for table " + t + " based on: " + cond);
pkid = db.cell(sql, this.Config.AliasTo);
}
exectimer["updateDecisionTime"] += (datetime.date() - insertUpdateStart);
if(t.toLowerCase() != "var")
{
if(this.DataType[t] == undefined) this.writeLog(this.LogLevels.Error, "Table " + t + " not exist in database");
else {
for(c in this.OutputRecord[t])
{
if(this.OutputRecord[t][c]["I+U"] != undefined && (this.OutputRecord[t][c]["I+U"] != "" || !this.skipEmptyValue) && this.OutputRecord[t][c]["I+U"] != null)
{
if(this.DataType[t][c] != undefined)
{
spalten.push(c);
typen.push(this.DataType[t][c]);
werte.push(this.OutputRecord[t][c]["I+U"]);
}
else
this.writeLog(this.LogLevels.Error, "Column " + c + " not exist in table " + t);
}
}
}
}
if(pkid != "0") // exists
{
if(t.toLowerCase() != "var")
{
if(this.DataType[t] == undefined) this.writeLog(this.LogLevels.Error, "Table " + t + " not exist in database");
else
{
for(c in this.OutputRecord[t])
{
if(this.OutputRecord[t][c]["U"] != undefined && (this.OutputRecord[t][c]["U"] != "" || !this.skipEmptyValue) && this.OutputRecord[t][c]["U"] != null
&& this.OutputRecord[t][c]["I+U"] == undefined)
{
if(this.DataType[t][c] != undefined)
{
spalten.push(c);
typen.push(this.DataType[t][c]);
werte.push(this.OutputRecord[t][c]["U"]);
}
else
this.writeLog(this.LogLevels.Error, "Column " + c + " not exist in table " + t);
}
}
}
}
if(!this.updateData(t, spalten, typen, werte, cond, this.Config.AliasTo)) skip = true;
}
else
{
if(t.toLowerCase() != "var")
{
if(this.DataType[t] == undefined) this.writeLog(this.LogLevels.Error, "Table " + t + " not exist in database");
else
{
for(c in this.OutputRecord[t])
{
if(this.OutputRecord[t][c]["I"] != undefined && this.OutputRecord[t][c]["I"] != "" && this.OutputRecord[t][c]["I"] != null
&& this.OutputRecord[t][c]["I+U"] == undefined)
{
if(this.DataType[t][c] != undefined)
{
spalten.push(c);
typen.push(this.DataType[t][c]);
werte.push(this.OutputRecord[t][c]["I"]);
}
else
this.writeLog(this.LogLevels.Error, "Column " + c + " not exist in table " + t);
}
}
}
}
if(!this.insertData(t, spalten, typen, werte, this.Config.AliasTo)) skip = true; }
exectimer["insertUpdateActions"] += (datetime.date() - insertUpdateStart);
}
// we do an insert
if(skip == false && this.Config.ImportCommand == "insert")
{
if(t.toLowerCase() != "var")
{
if(this.DataType[t] == undefined) this.writeLog(this.LogLevels.Error, "Table " + t + " not exist in database");
else
{
for(c in this.OutputRecord[t])
{
if(this.DataType[t][c] != undefined)
{
spalten.push(c);
typen.push(this.DataType[t][c]);
werte.push(this.OutputRecord[t][c]["I"]);
}
else
this.writeLog(this.LogLevels.Error, "Column " + c + " not exist in table " + t);
}
}
}
if(!this.insertData(t, spalten, typen, werte, this.Config.AliasTo)) skip = true;
}
// we do an update?
if(skip == false && this.Config.ImportCommand == "update")
{
if(t.toLowerCase() != "var")
{
if(this.DataType[t] == undefined) this.writeLog(this.LogLevels.Error, "Table " + t + " not exist in database");
else
{
for(c in this.OutputRecord[t])
{
if(this.DataType[t][c] != undefined)
{
spalten.push(c);
typen.push(this.DataType[t][c]);
werte.push(this.OutputRecord[t][c]["U"]);
}
else
this.writeLog(this.LogLevels.Error, "Column " + c + " not exist in table " + t);
}
}
}
// create condition by looking for key fields
cond = this.generateKeyCondition(t);
if(!this.updateData(t, spalten, typen, werte, cond, this.Config.AliasTo)) skip = true;
}
}
exectimer.outputPerRecordTime += datetime.date() - outputRecordTimeStart;
}
// this is *not* the else-branch for the if above, because we may change the value for "skip"
// even it was false when comparing in the "if" above. so we check this here in a separate
// if to get all occurrences of skipped import rows.
if(skip == true) this.recordCounts.skip++;
//Now insert/update all Columns
if(!skip)
{
var totalDBexecTime = datetime.date();
let insertSuccessfull = true;
let updateSuccessfull = true;
try
{ if(this.insertArray.length > 0)
{
insertSuccessfull = false;
var startInsert = datetime.date();
db.inserts(this.insertArray, this.Config.AliasTo);
exectimer.dbInsertTime += datetime.date() - startInsert;
this.recordCounts.insert++;
insertSuccessfull = true;
// insert
recordStack.insertUpdate = "insert";
recordStack.insertData = {
successful: true,
array: this.insertArray
};
}
if(this.updateArray.length > 0)
{
updateSuccessfull = false;
var startUpdate = datetime.date();
db.updates(this.updateArray, this.Config.AliasTo);
exectimer.dbUpdateTime += datetime.date() - startUpdate;
this.recordCounts.update++;
updateSuccessfull = true;
// update
recordStack.insertUpdate = "update";
recordStack.updateData = {
successful: true,
array: this.updateArray
};
}
}
catch(ex)
{
this.writeLog(this.LogLevels.Error, "Error at " + (insertSuccessfull ? "" : "Insert") + (updateSuccessfull ? "": "Update") + ":" + logging.toLogString(ex));
logging.log(ex["rhinoException"] != undefined ? ex["rhinoException"] : ex)
this.recordCounts.skip++;
recordStack.exception = logging.toLogString(ex["rhinoException"] != undefined ? ex["rhinoException"] : ex, true);
if(this.insertArray.length > 0 && !insertSuccessfull) {
this.writeLog(this.LogLevels.Info, "Insert array: " + JSON.stringify(this.insertArray, null, " "));
recordStack.insertData = {
successful: false,
array: this.updateArray
};
}
if(this.updateArray.length > 0 && !updateSuccessfull) {
this.writeLog(this.LogLevels.Info, "Update array: " + JSON.stringify(this.updateArray, null, " "));
recordStack.updateData = {
successful: false,
array: this.updateArray
};
}
}
exectimer.dbDataModifiedTime += datetime.date() - totalDBexecTime;
//Clear Arrays
this.insertArray = [];
this.updateArray = [];
}
// if a progress callback function has been defined, call it
if(this.ProgressCallback != null && typeof(this.ProgressCallback) == "function")
{
this.ProgressCallback(skip, bi, recordStack, this.Config.AliasFrom, this.Config.Callback.TableFrom, this.Config.Callback.IDColumn, this.Config.Callback.IDColumnIndex);
}
if(this.MaxRows > 0 && this.recordCounts.total >= this.MaxRows)
{
break;
} } // end for (row of current batch)
if(this.MaxRows > 0 && this.recordCounts.total >= this.MaxRows)
{
break;
}
exectimer.dataLoopTime += datetime.date() - dataLoopTimeStart;
}
this.showCounts();
stopTime = datetime.date();
this.showTimings();
s = datetime.toDate(stopTime, translate.text("yyyy-MM-dd HH:mm:ss"), "UTC");
this.writeLog(this.LogLevels.Minimal, "End at " + s + " UTC");
logging.log("[IMPORT] End at " + s + " UTC");
return {
totalCount: this.recordCounts.total
,skipCount: this.recordCounts.skip
,insertCount: this.recordCounts.insert
,updateCount: this.recordCounts.update
,unchangedCount: this.recordCounts.unchanged
};
}
// show timing information
this.showTimings = function(pPrefix)
{
if (pPrefix == undefined)
pPrefix = "";
var mappingtotal = 0;
for(var i=0; i < this.Config.Mapping.length; i++)
{
mappingtotal += mappingtimer[i];
}
if (stopTime == undefined)
this.writeLog(this.LogLevels.Warning, pPrefix + "Total run time till now: " + ((datetime.date() - startTime) / 1000) + " seconds.");
else
this.writeLog(this.LogLevels.Warning, pPrefix + "Total run time: " + ((stopTime - startTime) / 1000) + " seconds.");
for(var k in exectimer)
{
this.writeLog(this.LogLevels.Info, pPrefix + "Total " + k + " time: " + (exectimer[k] == "0" ? "<1" : exectimer[k]) + " ms ");
}
this.writeLog(this.LogLevels.Warning, pPrefix + "Total map time: " + (mappingtotal / 1000) + " seconds.");
for(i=0; i < this.Config.Mapping.length; i++)
{
var fname = this.Config.Mapping[i][0].name;//name is inherited function-property
this.writeLog(this.LogLevels.Info, pPrefix + "Mapping #" + i + " " + fname + ": " + (mappingtimer[i] == "0" ? "<1" : mappingtimer[i]) + " ms ");
}
}
this.showCounts = function(pPrefix)
{
if (pPrefix == undefined)
pPrefix = "";
this.writeLog(this.LogLevels.Minimal, pPrefix + "Total: " + this.recordCounts.total
+ ", Skipped: " + this.recordCounts.skip
+ ", Inserted: " + this.recordCounts.insert
+ ", Updated: " + this.recordCounts.update
+ ", Unchanged: " + this.recordCounts.unchanged );
}
// this function yields the next batch of data to import.
this.getNextBatch = function(pConfig){
var tt = datetime.date(); // exec timer
var resultBatch;
var bs; var daten;
this.writeLog(this.LogLevels.Debug, "Executing getNextBatch()");
batchNum++;
var impmode;
if(pConfig.DataFunction != undefined)
impmode = "array";
else if(pConfig.DataFile != undefined)
{
if(pConfig.DataFile.substr(pConfig.DataFile.length-3, pConfig.DataFile.length) == 'xml') //is the file an xml file? then use xml-mode
impmode = "xml";
else
impmode = "file";
}
else if(pConfig.XMLObject != undefined)
impmode = "xml";
else
impmode = "table";
switch(impmode)
{
case "xml":
if(batchNum > 1) return null; // we read files in one sweep
if(this.Debug == true) this.writeLog("Getting input xml " + pConfig.DataFile, false, 3);
// file import, which is always a single batch operation
importtype = "F";
var strXML;
// get xml-string directly or load from file?
if(pConfig.DataFile != "")
strXML = this.getFileContent(pConfig.DataFile, util.DATA_TEXT); //load from file
else
strXML = pConfig.XMLObject; //get xml-string
if(strXML != "" && strXML != undefined)
{
var xmlData = new XML(strXML);
resultBatch = [];
var xmlArr = [];
//get every row-element
for each(xmlItem in xmlData.data.row) // TODO: replace "for each (.. in ..)" with for (.. of ..) when the designer supports it (the server already supports it). As it's deprecated in js https://developer.mozilla.org/de/docs/Web/JavaScript/Reference/Statements/for_each...in
{
xmlArr = [];
for each(xmlItem2 in xmlItem.column)
xmlArr.push(xmlItem2);
resultBatch.push(xmlArr);
}
}
else
this.writeLog("XML-Data is empty or undefined!", false, 0);
return resultBatch;
break;
case "file":
if(batchNum > 1) return null; // we read files in one sweep
if(this.Debug == true) this.writeLog("Reading input file " + pConfig.DataFile, false, 3);
// file import, which is always a single batch operation
importtype = "F";
// otherwise, load the file
var filestring = this.getFileContent(pConfig.DataFile, util.DATA_TEXT);
var rs = pConfig.RowSeparator; if(rs == undefined) rs = "\n";
var cs = pConfig.ColumnSeparator;
if(cs == undefined) cs = ",";
var sd = pConfig.StringDelimiter;
if(sd == undefined) sd = "";
resultBatch = text.parseCSV(filestring, rs, cs, sd);
tt = datetime.date() - tt;
exectimer["getdata"] += tt;
return resultBatch;
break;
case "array":
bs = this.BatchSize;
if(bs == undefined) bs = 0;
resultBatch = this.Config.DataFunction.call(this, batchNum, bs);
tt = datetime.date() - tt;
exectimer["getdata"] += tt;
return resultBatch;
break;
case "table":
// table import
bs = this.BatchSize;
if(bs == undefined)
bs = 0;
//bs specified: read data in blocks to save memory
if(bs > 0 )
{
if(this.Config.IdQuery == undefined || this.Config.IdColumn == undefined)
{
//new method for batch-processsing (paging)
daten = db.tablePage(this.Config.DataQuery, this.Config.AliasFrom, batchStart, bs);
batchStart += bs;
//this happens when all-records % pBlockSize == 0
//we do not want to call the callback-fn
if (daten.length == 0)
daten = null;
}
else
{
if (this.idList == undefined)
{
this.writeLog(this.LogLevels.Warning, "using \"IdQuery\" and \"IdColumn\" is an obsolte method for getting sql-data in blocks");
this.idList = this.createIDList();
}
//legacy method for batch-processsing
if(batchStart < this.idList.length) // as long as we got something to do
{
batchList = this.idList.slice(batchStart, batchStart + bs);
batchStart += bs;
// get data of the next batch
var sql = this.Config.DataQuery.replace(/\$\$id/i, idcolumn + " in ('" + batchList.join("','") + "')" );
daten = db.table(sql, this.Config.AliasFrom);
}
else
{
this.idList = undefined;
daten = null;
}
}
}
else
{
// return complete result set if(batchNum > 1)
daten = null;
else
daten = db.table(this.Config.DataQuery, this.Config.AliasFrom);
}
tt = datetime.date() - tt;
exectimer["getdata"] += tt;
return daten;
break;
}
}
this.createIDList = function()
{
var tt = datetime.date(); // exectimer
var resIDList;
if(pConfig.IdQuery != undefined && pConfig.DataQuery != undefined && pConfig.IdColumn != undefined)
{
// set the id column for the batch retrieval
idcolumn = this.Config.IdColumn;
// set multi-batch import
importtype = "M";
// get the list of primary keys
resIDList = db.array(db.COLUMN, pConfig.IdQuery, pConfig.AliasFrom);
tt = datetime.date() - tt;
exectimer["getdata"] += tt;
}
return resIDList;
}
// @param String pMessage -- die Meldung, die geloggt werden soll
// @param String pLevel
this.writeLog = function(pLevel, pMessage)
{
var tt = datetime.date();
// if logging has been "oursourced", just call the callback function and do nothing
if(this.LogCallback != null && typeof(this.LogCallback) == "function")
{
this.LogCallback(pLevel, pMessage);
}
else if(pLevel <= this.LogLevel) // shall we output this message?
{
var logprefix = "[IMPORTER]@" + datetime.toDate(datetime.date(), translate.text("yyyy-MM-dd HH:mm:ss"), "UTC") + " UTC: ";
var logline = logprefix + pMessage;
if (this.enableLogBuffer)
logBuffer.push(logline);
if(this.Log == "LOGFILE")
{
logging.log(logline);
}
else
{
if(this.Log == "CONSOLE")
{
logging.log(logline);
}
else if(this.Log != "" && this.Log != undefined)
{
// log in globalvar
if(this.Log.split(".")[0].toLowerCase() == "$global")
{
var s = logline + "\r\n";
vars.set(this.Log, vars.getString(this.Log) + s);
}
}
} }
tt = datetime.date() - tt;
exectimer["logging"] += tt;
}
// retrieve all log messages currently in log buffer
this.getLogMessages = function(){
return logBuffer;
}
// return true, if running in client context, false for a server context
this.isClientProcess = function(){
return vars.getString("$sys.isclient") == "true";
}
this.getFileContent = function(pFilename, pDataType)
{
var resContent;
var cp;
if(this.isClientProcess() == true) // use doClientIntermediate
{
try
{
resContent = swing.doClientIntermediate(swing.CLIENTCMD_GETDATA, [pFilename, pDataType, this.fileInputCharset]);
}
catch(ex)
{
logging.show(ex)
resContent = "";
}
}
else // use fileIO
{
try
{
resContent = fileIO.getData(pFilename, util.DATA_TEXT, this.fileInputCharset);
}
catch(ex)
{
logging.log( ex );
resContent = "";
}
}
return resContent;
}
this.dumpRecord = function(pTable, pColumns, pTypes, pValues, pCondition)
{
var PADDING = "................................";
var s = "";
if(pCondition != undefined) s += "update " + pTable + " where " + pCondition; else s += "insert " + pTable;
s += "\n";
for(var i=0; i < pColumns.length; i++)
{
s += " " + pColumns[i] + PADDING.substr(0, 32- pColumns[i].length) + ": |" + pValues[i] + "|\n";
}
s += "\n";
return s;
}
this.insertData = function(pTable, pColumns, pTypes, pValues, pAlias)
{
var tt = datetime.date(); var resData = true;
if(this.Preview == false)
{
//better safe than sorry ...
try
{
if(pTable.toLowerCase() != "var")
{
if(this.DataType[pTable] == undefined) this.writeLog(this.LogLevels.Error, "Table " + pTable + " not exist in database");
else
{
this.writeLog(this.LogLevels.Debug, "INSERT for [" + pAlias + "].[" + pTable + "]");
this.writeLog(this.LogLevels.Preview, this.dumpRecord(pTable, pColumns, pTypes, pValues));
//exist already "USER_NEW" and/or "DATE_NEW" in the columns resultset?
var uNew = false;
var dNew = false;
for(var i = 0; i < pColumns.length; i++)
{
if(uNew == false && pColumns[i].toUpperCase() == "USER_NEW")
uNew = true;
if(dNew == false && pColumns[i].toUpperCase() == "DATE_NEW")
dNew = true;
}
// process audit columns automagically
if(uNew == false && this.DataType[this.getTableCase(pTable)][this.getColumnCase("USER_NEW")] != undefined)
{
pColumns.push(this.getColumnCase("USER_NEW"));
if (pTypes != null) pTypes.push(this.DataType[this.getTableCase(pTable)][this.getColumnCase("USER_NEW")]);
pValues.push(this.ImportUser);
}
if(dNew == false && this.DataType[pTable][this.getColumnCase("DATE_NEW")] != undefined)
{
pColumns.push(this.getColumnCase("DATE_NEW"));
if(pTypes != null) pTypes.push(this.DataType[this.getTableCase(pTable)][this.getColumnCase("DATE_NEW")]);
pValues.push(datetime.date());
}
this.insertArray.push([this.getTableCase(pTable), pColumns, pTypes, pValues, pAlias]);
}
}
}
catch(ex)
{
this.writeLog(this.LogLevels.Error, "Exception at insertData for record: " + getRecordString(pColumns, pValues));
logging.log(ex);
resData = false;
}
}
else
{
this.writeLog(this.LogLevels.Preview, "Insert into table " + pTable);
this.writeLog(this.LogLevels.Preview, this.dumpRecord(pTable, pColumns, pTypes, pValues));
}
tt = datetime.date() - tt;
exectimer["insertData"] += tt;
return resData;
}
this.updateData = function(pTable, pColumns, pTypes, pValues, pCondition, pAlias)
{
var tt = datetime.date();
var theCols;
var theTypes;
var theValues;
var resultData = true;
if(this.Preview == false)
{ try
{
if(pTable.toLowerCase() != "var")
{
if(this.DataType[pTable] == undefined) this.writeLog("Table " + pTable + " not exist in database");
else
{
this.writeLog(this.LogLevels.Debug, "UPDATE for alias [" + pAlias + "].[" + pTable + "]");
this.writeLog(this.LogLevels.Preview, this.dumpRecord(pTable, pColumns, pTypes, pValues, pCondition));
if (this.CompleteUpdate == false) // check for changed database values and use only changed columns for update
{
var uColumns = new Array();
var uValues = new Array();
var uTypes = new Array();
var oldValues = db.array(db.ROW, "select " + pColumns.join(", ") + " from " + pTable + " where " + pCondition, pAlias);
for (var dsi = 0; dsi < oldValues.length; dsi++)
{
if (oldValues[dsi] != pValues[dsi])
{
//get the values from the validate target
uColumns.push(pColumns[dsi]);
uValues.push(pValues[dsi]);
if (pTypes != null) uTypes.push(pTypes[dsi]);
}
}
theCols = uColumns;
theTypes = uTypes;
theValues = uValues;
}
else // update all columns, so use default column set
{
theCols = pColumns;
theTypes = pTypes;
theValues = pValues;
}
if(theCols.length > 0)
{
//show the old and the new data only if anything changed
this.writeLog(this.LogLevels.Preview, "New Data: " + pValues.join("|"));
if (oldValues != undefined)
this.writeLog(this.LogLevels.Preview, "Old Data: " + oldValues.join("|"));
}
var minchanges = 0;
var dEdit = false;
var uEdit = false;
for(var i = 0; i < theCols.length; i++)
{
if(uEdit == false && theCols[i].toUpperCase() == "USER_EDIT")
uEdit = true;
if(dEdit == false && theCols[i].toUpperCase() == "DATE_EDIT")
dEdit = true;
}
// process audit columns automagically
if(uEdit == false && this.DataType[this.getTableCase(pTable)][this.getColumnCase("USER_EDIT")] != undefined)
{
theCols.push(this.getColumnCase("USER_EDIT"));
if(pTypes != null) theTypes.push(this.DataType[this.getTableCase(pTable)][this.getColumnCase("USER_EDIT")]);
theValues.push(this.ImportUser);
minchanges++;
}
if(dEdit == false && this.DataType[this.getTableCase(pTable)][this.getColumnCase("DATE_EDIT")] != undefined)
{
theCols.push(this.getColumnCase("DATE_EDIT")); if(pTypes != null) theTypes.push(this.DataType[this.getTableCase(pTable)][this.getColumnCase("DATE_EDIT")]);
theValues.push(datetime.date());
minchanges++;
}
if(this.CompleteUpdate == false)
{
if(theCols.length > minchanges)
{
this.updateArray.push([this.getTableCase(pTable), theCols, theTypes, theValues, pCondition, pAlias]);
}
}
else
{
this.updateArray.push([this.getTableCase(pTable), theCols, theTypes, theValues, pCondition, pAlias]);
}
}
}
}
catch(ex)
{
resultData = false;
this.writeLog(this.LogLevels.Error, "Exception at updateData for record: " + getRecordString(pColumns, pValues));
logging.log(ex);
}
}
else
{
this.writeLog(this.LogLevels.Preview, "Update table " + pTable);
this.writeLog(this.LogLevels.Preview, this.dumpRecord(pTable, pColumns, pTypes, pValues, pCondition));
}
tt = datetime.date() - tt;
exectimer["updateData"] += tt;
return resultData;
}
// set default action for a mapping call, if action has not been specified
this.setDefaultAction = function(pObject)
{
if(pObject.Action == undefined) // set reasonable defaults for Action, if not specified
{
if(this.Config.ImportCommand == "insert") pObject.Action = "I";
else
if(this.Config.ImportCommand == "update") pObject.Action = "U";
else
if(this.Config.ImportCommand == "insert+update") pObject.Action = "I+U";
}
}
// ATTENTION!! This is the *new* version and not the same as resolveSymbols!!
//
// resolve symbol to get import data
// may contain literals string and {#} and {tbl.col} symbols
// if undefined or empty expression is provided, return an empty string
this.resolveSymbol = function(pObject, pExpression, pEvalScript)
{
var expr;
if((pExpression != undefined) && (pExpression != ""))
{
var inp = this.InputRecord;
var out = this.OutputRecord;
var cCase = this.ColumnCase;
var tCase = this.TableCase;
var uCase = this.Cases.Upper;
this.setDefaultAction(pObject);
var obj = pObject;
let self = this;
expr = pExpression.toString();
expr = expr.replace(/\{([._a-zA-Z0-9]+)\}/ig,
function(pMatch, pNumber)
{
if(isNaN(Number(pNumber)))
{
var varname = pNumber.split(".");
var res;
// action verwenden, wenn keine da => importcommand auslesen
var action = obj.Action;
if(out[ self.getTableCase(varname[0]) ] != undefined && out[ self.getTableCase(varname[0]) ][ self.getColumnCase(varname[1]) ] != undefined)
{
switch(obj.Action)
{
case "I":
res = out[ self.getTableCase(varname[0]) ][ self.getColumnCase(varname[1]) ]["I"];
break;
case "U":
res = out[ self.getTableCase(varname[0]) ][ self.getColumnCase(varname[1]) ]["U"];
break;
case "I+U":
res = out[ self.getTableCase(varname[0]) ][ self.getColumnCase(varname[1]) ]["U"]; //2009-06-16 TR
if(res == undefined || res == "") res = out[ self.getTableCase(varname[0])][ self.getColumnCase(varname[1]) ]["I"]; //2009-06-16 TR
break;
}
}
else // varname does not exist as a property of out
{
res = undefined;
}
if(res == undefined) res = ""; // blank out NULL values
//transform ' to \\u0027 which is in eval a \u0027; transform " to \\u0022 which is in eval a \u0022
//needed for eval like: "'3'2' != ''"
//also escape LF and CR to prevent unterminated string errors
//if the transform of LF and CR cause problems:
//- remove the 2 replaces
//- document in the comment why it should not be used and when it will cause other problems
//- find another solution to prevent the unterminated string errors
if(pEvalScript)
return res.replace("'", "\\\\u0027", "g").replace("\"", "\\\\u0022", "g").replace("\r", "\\\\u000D", "g").replace("\n", "\\\\u000A", "g");
else
return res;
}
else
{
//transform ' to \\u0027 which is in eval a \u0027; transform " to \\u0022 which is in eval a \u0022
//needed for eval like: "'3'2' != ''"
//also escape LF and CR to prevent unterminated string errors
//if the transform of LF and CR cause problems:
//- remove the 2 replaces
//- document in the comment why it should not be used and when it will cause other problems
//- find another solution to prevent the unterminated string errors
if(pEvalScript)
return inp[Number(pNumber)].replace("'", "\\\\u0027", "g").replace("\"", "\\\\u0022", "g").replace("\r", "\\\\u000D", "g").replace("\n", "\\\\u000A", "g");
else
return inp[Number(pNumber)];
}
} );
if(pEvalScript == true)
expr = eval(expr);
}
else
{
expr = "";
}
return expr; }
// read column type information for all columns in all tables of the alias specified
this.getDataTypes = function(pAlias)
{
var tables = db.getTables(pAlias);
var dataTypes = new Object();
for(var i=0; i < tables.length; i++)
{
if (tables[i] != "trace_xe_action_map" && tables[i] != "trace_xe_event_map")
{
var cols = db.getColumns(tables[i], pAlias);
var types = db.getColumnTypes(tables[i], cols, pAlias);
dataTypes[ tables[i] ] = new Object(); // create sub-object to hold columns
for(var j=0; j < cols.length; j++) dataTypes[tables[i]][cols[j]] = types[j];
}
}
return dataTypes; // return object with type information
}
// sets the output record buffer according to "Action" performed
this.setOutput = function(pObject, pValue)
{
try
{
var target = pObject.Target.split(".");
this.setDefaultAction(pObject);
// make sure we do have an output buffer
if(this.OutputRecord[ this.getTableCase(target[0]) ] == undefined) this.OutputRecord[ this.getTableCase(target[0]) ] = new Object();
if(this.OutputRecord[ this.getTableCase(target[0]) ][ this.getColumnCase(target[1]) ] == undefined) this.OutputRecord[ this.getTableCase(target[0]) ][ this.getColumnCase(target[1]) ] = new Object();
switch(pObject.Action)
{
case "I":
this.OutputRecord[ this.getTableCase(target[0]) ][ this.getColumnCase(target[1]) ]["I"] = pValue;
break;
case "U":
this.OutputRecord[ this.getTableCase(target[0]) ][ this.getColumnCase(target[1]) ]["U"] = pValue;
break;
case "I+U":
this.OutputRecord[ this.getTableCase(target[0]) ][ this.getColumnCase(target[1]) ]["I"] = pValue; //2009-06-16 TR
this.OutputRecord[ this.getTableCase(target[0]) ][ this.getColumnCase(target[1]) ]["U"] = pValue; //2009-06-16 TR
this.OutputRecord[ this.getTableCase(target[0]) ][ this.getColumnCase(target[1]) ]["I+U"] = pValue; //2009-06-16 TR
break;
}
}
catch(ex)
{
this.writeLog(this.LogLevels.Error, "Property <Target> not set for mapping object!");
logging.log(ex);
logging.log("[IMPORTER] Property <Target> not set for mapping object!");
}
}
// get the content of the output record buffer according to "Action" performed
this.getOutput = function(pObject, pTarget)
{
var target;
var action;
target = pTarget.split(".");
if(pObject != undefined) action = pObject.Action; else action = undefined;
if(action == undefined) // set reasonable defaults for Action, if not specified
{
if(this.Config.ImportCommand == "insert") action = "I";
else
if(this.Config.ImportCommand == "update") action = "U";
else
if(this.Config.ImportCommand == "insert+update") action = "I+U";
}
var resAction;
switch(action) {
case "I":
resAction = this.OutputRecord[ this.getTableCase(target[0]) ][ this.getColumnCase(target[1])]["I"];
break;
case "U":
resAction = this.OutputRecord[ this.getTableCase(target[0]) ][ this.getColumnCase(target[1]) ]["U"];
break;
case "I+U":
resAction = this.OutputRecord[ this.getTableCase(target[0]) ][ this.getColumnCase(target[1]) ]["U"];
if(resAction == undefined || resAction == "") resAction = this.OutputRecord[ this.getTableCase(target[0]) ][ this.getColumnCase(target[1]) ]["I"];
break;
}
if(resAction == undefined) resAction = ""; // blank out undefined
return resAction;
}
// generates condition clause (without "WHERE") to check for existing key column values
this.generateKeyCondition = function(pTable)
{
var whereclause = "";
for(var col in this.KeyColumn[pTable])
{
var tmp = this.KeyColumn[pTable][col]; // contains I | I;U | I+U
if(tmp != "")
{
var value = this.OutputRecord[pTable][col]["U"];
if(value == undefined || value == "")
value = this.OutputRecord[pTable][col]["I+U"];
if(value == undefined || value == "")
value = this.OutputRecord[pTable][col]["I"];
// we cannot distinguish between an empty string and NULL in Jdito,
// so if get an empty string for the value, we do a check for empty string OR NULL
if(value == "")
{
whereclause += " and (" + col + " = '' OR " + col + " is null)";
}
else
{
whereclause += " and " + col + " = '" + db.quote(value, this.Config.AliasTo) + "'";
}
}
}
// remove leading "and" from where clause
if(whereclause.substr(0, 4) == " and") whereclause = whereclause.substr(4, whereclause.length);
return whereclause;
}
/*
* yield a string representation of the record in pColumns and pValues for logging
* purposes and debugging.
*
* @param {String[]} pColumns req columns that are represented
* @param {String[]} pValues req values that are represented
*
* @return {String}
*/
function getRecordString(pColumns, pValues)
{
var s = "";
for(var i=0; i < pColumns.length; i++)
{
var v = "";
if(pValues[i]) v = pValues[i].toString();
if(v.length > 40) v = v.substr(0,39) + "...";
s += " COL: " + pColumns[i] + ": " + v + "\n";
}
return s; }
//Converts a string with the tablename in upper or lower case
this.getTableCase = function(pName)
{
if(this.TableCase == this.Cases.Upper)
return pName.toUpperCase();
else if (this.TableCase == this.Cases.Lower)
return pName.toLowerCase();
else
return pName;
}
//Converts a string with the columnname in upper or lower case
this.getColumnCase = function(pName)
{
if (this.ColumnCase == this.Cases.Upper)
return pName.toUpperCase();
else if (this.ColumnCase == this.Cases.Lower)
return pName.toLowerCase();
else
return pName;
}
this.doIfCheck = function(pObject)
{
if (pObject.DoIf == undefined) return true;
if (typeof(pObject.DoIf) == "function")
{
var expr;
if(pObject.Source != undefined) expr = this.InputRecord[pObject.Source];
if(pObject.Value != undefined) expr = this.resolveSymbol(pObject, pObject.Value, pObject.Eval);
if(pObject.Map != undefined && pObject.Index) expr = pObject.Map[this.resolveSymbol(pObject, pObect.Index, pObject.Eval)];
return pObject.DoIf.call(this, this.InputRecord, expr, pObject);
}
return (this.resolveSymbol(pObject, pObject.DoIf, true));
}
}
/**
* Data handler for csv paging
*
* @param pCurrentBatchNum
* @param pBatchSize
*/
function batchCsvLoad(pCurrentBatchNum, pBatchSize)
{
if (this.BatchStart == undefined) this.BatchStart = 0;
if (this.Config.RowSeparator != undefined) this.writeLog(this.LogLevels.Warning, "'RowSeparator' option is currently not supported!")
var dataStr = "";
var fn = this.Config.DataFile;
var bs = pBatchSize;
if (!fileIO.exists(fn))
throw new Error(translate.withArguments("file '%0' does not exist or you have got no permission on this file", [fn]));
else if (!fileIO.canRead(fn))
throw new Error(translate.withArguments("file '%0' cannot be read", [fn]));
const MAX_LOOP_SIZE = this.Config.BatchSize;
do
{
dataStr = null;
var loopSize = MAX_LOOP_SIZE;
try
{
dataStr = fileIO.getBulkData(fn, util.DATA_TEXT, this.fileInputCharset, this.BatchStart, bs);
}
catch(ex)
{
break; }
this.BatchStart += bs;
whileMultiline: while (dataStr.slice(-2) != "\r\n" && (dataStr.slice(-1) == "\n" || dataStr.slice(-1) == "\r"))//we have to load more Data
{
try
{
dataStr += fileIO.getBulkData(fn, util.DATA_TEXT, this.fileInputCharset, this.BatchStart, 1);
}
catch(ex)
{
this.writeLog(this.LogLevels.Warning, ex);
break;
}
this.BatchStart++;
if (loopSize == 0)
{
this.writeLog(this.LogLevels.Error, "MAX_LOOP_SIZE reached; Abort");
break whileMultiline;
}
loopSize--;
}
if (dataStr.slice(-2) == "\r\n")
dataStr = dataStr.slice(0, -2);
var rs = this.Config.RowSeparator;
if(rs == undefined) rs = "\r\n";
var cs = this.Config.ColumnSeparator;
if(cs == undefined) cs = ";";
var sd = this.Config.StringDelimiter;
if(sd == undefined) sd = "";
return text.parseCSV(dataStr, rs, cs, sd);
}
while(dataStr != null);
return null;
}