1 /* *****************************************************************************
  2  *	ITSV GmbH
  3  *
  4  *	Software Operations
  5  *	Zentrale Daten und Services
  6  *
  7  *	Product:	CCDB
  8  *	Module:		auxiliary
  9  *	History:
 10  *	Date       |  Author   | Version    | DESCRIPTION
 11  *  -----------+-----------+------------+-------------------------------------------
 12  *  04.11.2015 | WSC       | 0.6        | extracted from ccdb.js
 13  *             |           |            |
 14  *             |           |            |
 15  *             |           |            |
 16  *             |           |            |
 17  *             |           |            |
 18  */
 19 
 20  /*
 21   *	node.js internal modules
 22   */
 23  var path = require('path');
 24  var fs   = require('fs');
 25  
 26 /*
 27  *	Dependencies from other modules
 28  */
 29 var flow = require('flow');
 30 var dateformat = require('dateformat');
 31 var util = require('util');
 32 var readChunk = require('read-chunk'); 					// npm install --save read-chunk - needed by getFileType
 33 var fileType = require('file-type');
 34 var XLSX = require('xlsx');
 35 var logger = null;
 36 var mastername = "ccdb.aux.mastername not set";			// mastername is to be set by the module initializion (at the end of this file)
 37 var prefs = {"default" : "auxiliarx.prefs not set"};	// prefs is to be set by the module initializion (at the end of this file) to point to the application global preferences object
 38 
 39 // ATTENTION: this is sort of a cyclic dependency, but should work !
 40 // var functions = require('./functions.js');
 41 /*
 42  * dateformat 'serialDateTime' formats date/time in one string which only has
 43  * numeric characters and can be used to sort chronologically
 44  *
 45  */
 46 dateformat.masks.serialDateTime = "yyyymmddHHMMssl";
 47 /*
 48  *	dateformat 'eutimedate' formats date/time in EU mode
 49  */
 50 dateformat.masks.euDateTime = "yyyy-mm-dd HH:MM:ss";
 51 
 52 /* ****************************************************************************
 53  *	FUNCTION:		DEC
 54  *	DESCRIPTION:	format number as decimal string
 55  *					string is padded with leading zeroes to <ndig> places
 56  *					if numeric string would be longer than <ndig> the string is
 57  *					cut down right-aligned to <ndig> places, higher order places
 58  *					to the left are cut off
 59  */  
 60 function DEC(num,ndig) {
 61 	ndig = ndig || 32;			// default to 32 digits, if <ndig> not supplied
 62 	var s = num.toString();		// convert number to decimal string
 63 	while (s.length<ndig) {		// left-pad string with zeroes up to <ndig> length
 64 		s = '0'+s;
 65 	}
 66 	if (s.length>ndig) {		// cut down to <ndig> length, negative index selects rightmost characters
 67 		s = s.slice(0-ndig);
 68 	}
 69 	return s;
 70 }
 71 
 72 /* ****************************************************************************
 73  *	FUNCTION:		escapeSingleQuote
 74  *	INPUT:			s 	-	string with single quotes (') to be escaped
 75  *	RESULT:			copy of <s> with all single quotes escaped (replaced by HTML-entity """)
 76  *	DESCRIPTION:	HTML-entity-escape all single quotes (') in <s>
 77  */
 78  function escapeSingleQuote(s) {
 79 	 // logger.debug("esq: BEFORE: >>"+s+"<<");
 80 	 var res = s.replace(/'/g,""");
 81 	 // logger.debug("esq: AFTER:  >>"+res+"<<");
 82 	 return res
 83  }
 84  
 85 /* ****************************************************************************
 86  *	FUNCTION:		escapeSingleQuoteAllStrings
 87  *	INPUT:			o	-	object
 88  *	DESCRIPTION:	escape single quotes in all strings that are part of <o>
 89  */
 90  function escapeSingleQuoteAllStrings(o) {
 91 	for (oa in o) {
 92 		if (typeof o[oa] === 'object') {
 93 			escapeSingleQuoteAllStrings(o[oa]);
 94 		} else if (typeof o[oa] === 'string') {
 95 			 o[oa] = escapeSingleQuote(o[oa]);
 96 		}
 97 	}
 98  } 
 99 
100 
101 /* ****************************************************************************
102  *	FUNCTION:		escapeDoubleQuote
103  */
104  function escapeDoubleQuote(s) {
105 	 // logger.debug("edq: BEFORE: >>"+s+"<<");
106 	 var res = s.replace(/"/g,"\\\"");
107 	 // logger.debug("edq: AFTER:  >>"+res+"<<");
108 	 return res
109  }
110  
111 /* ****************************************************************************
112  *	FUNCTION:		escapeDoubleQuoteAllStrings
113  */
114  function escapeDoubleQuoteAllStrings(o) {
115 	for (oa in o) {
116 		if (typeof o[oa] === 'object') {
117 			escapeDoubleQuoteAllStrings(o[oa]);
118 		} else if (typeof o[oa] === 'string') {
119 			 o[oa] = escapeDoubleQuote(o[oa]);
120 		}
121 	}
122  } 
123  
124  /* ****************************************************************************
125   *	FUNCTION:		replaceControlWhitespace
126   * DESCRIPTION:	replaces all non-printable control-codes in <s> with a single space character
127   */
128  function replaceControlWhitespace(s) {
129 	 // logger.debug("rcw: BEFORE: >>"+s+"<<");
130 	 var res = s.replace(/[\x00-\x1F\x7F-\x9F]/gi,' ');
131 	 // logger.debug("rcw: AFTER:  >>"+res+"<<");
132 	 return res
133  }
134  
135 /* ****************************************************************************
136  *	FUNCTION:		replaceControlWhitespaceAllStrings
137  */
138  function replaceControlWhitespaceAllStrings(o) {
139 	for (oa in o) {
140 		if (typeof o[oa] === 'object') {
141 			replaceControlWhitespaceAllStrings(o[oa]);
142 		} else if (typeof o[oa] === 'string') {
143 			 o[oa] = replaceControlWhitespace(o[oa]);
144 		}
145 	}
146  }
147 
148 /* *****************************************************************************
149  *	FUNCTION:		objectIsEmpty
150  *	INPUT:			o	-	the object to be checked
151  *	RESULT:			TRUE if <o> has no properties
152  *	DESCRIPTION:	check is <o> is empty by enumerating all properties
153  *	ATTENTION:		needs ECMAscript5-compatible JavaScript-Engine!!
154  */
155  function objectIsEmpty(o) {
156 	return Object.getOwnPropertyNames(o).length===0;
157  }
158  
159 /* *****************************************************************************
160  * 	FUNCTION:	dateFromSerialString
161  *	INPUT:		S		-	string with date in form YYYYMMDDhhmmssttt
162  *					YYYY	-	Year 4 digits
163  *					MM		-	Month 2 digits
164  *					DD		-	Day 2 digits
165  *					hh		-	Hour 2 digits
166  *					mm		-	Minutes 2 digits
167  *					ss		-	Seconds 2 digits
168  *					ttt		-	Seconds fraction 3 digits
169  *	RESULT:		date representing S
170  *	DESCRIPTION:	decodes a Javascript-Date from S
171  *					if parts are missing, assume defaults:
172  *					Year:	1900
173  *					Month:	01
174  *					Day:	01
175  *					Time:	00:00:00.000
176  */
177 function dateFromSerialString (s) {
178 	var yr = 1900;
179 	var mo = 01;
180 	var da = 01;
181 	var hr = 0;
182 	var mi = 0;
183 	var sc = 0;
184 	var ts = 0;
185 	if (s.length>14) ts = parseInt(s.substring(14,17));
186 	if (s.length>12) sc = parseInt(s.substring(12,14));
187 	if (s.length>10) mi = parseInt(s.substring(10,12));
188 	if (s.length>08) hr = parseInt(s.substring(08,10));
189 	if (s.length>06) da = parseInt(s.substring(06,08));
190 	if (s.length>04) mo = parseInt(s.substring(04,06));
191 	if (s.length>00) yr = parseInt(s.substring(00,04));
192 	return new Date(yr,mo,da,hr,mi,sc,ts);
193 }
194 
195  /* ****************************************************************************
196   * FUNCTION:	dateFromWeekNr
197   * INPUT:		YR	- full year number
198   *				KW	- week number
199   * RETURNS:	Date object representing the MONDAY of the week YR/KW
200   *				number of week is assumed to be according to ISO 8601:
201  *				- week 1 is the week containing the 4-JAN
202   */
203 function dateFromWeekNr(yr,kw) {
204 	var msperday = 24*60*60*1000;		// milliseconds per day
205 	var jan4 = new Date(yr,0,4);		// JAN 4 is positively in week 1	
206 	var jan4day = jan4.getDay(); 		// day number of JAN-04 (SUN=0)
207 	jan4day = (jan4day==0)?7:jan4day;	// day number of JAN-04 (SUN=7)
208 	var diff2mon = jan4day-1;			// backwards difference from JAN-04 to monday of week 1 in days
209 	return new Date(jan4-(diff2mon*msperday)+((kw-1)*7*msperday));
210 }
211 
212 /* *****************************************************************************
213  * FUNCTION:	getWeekNr
214  * INPUT:		D - Date object
215  * RETURNS:		number of week according to ISO 8601:
216  *				- week 1 is the week containing the 4-JAN
217  */
218 function getWeekNr( d ) {
219 	// Create a copy of this date object
220 	var target = new Date(d.valueOf());
221 	// ISO week date weeks start on monday (mon=0, tue=1, wed=2...), javascript week-days on sunday (sun=0,mon=1, ...)
222 	// so correct the day number
223 	var dayNr = (d.getDay() + 6) % 7;
224 	// Set the target to the thursday of this week so the
225 	// target date is in the right year
226 	target.setDate(target.getDate() - dayNr + 3);
227 	// ISO 8601 states that week 1 is the week
228 	// with january 4th in it
229 	var jan4 = new Date(target.getFullYear(), 0, 4);
230 	// Number of days between target date and january 4th
231 	var dayDiff = (target - jan4) / 86400000;
232 	// Calculate week number: Week 1 (january 4th) plus the
233 	// number of weeks between target date and january 4th
234 	var weekNr = 1 + Math.ceil(dayDiff / 7);
235 	return weekNr;
236 }
237 
238  
239 /* ****************************************************************************
240  *	FUNCTION:		nowstring
241  *	DESCRIPTION:	returns the current date and time formatted in accordance
242  *					with datetime-format "serialDateTime"
243  */
244 function nowstring() {
245 	return dateformat(new Date(),"serialDateTime");
246 }
247 
248 /* ****************************************************************************
249  *	FUNCTION:		noweutime
250  *	DESCRIPTION:	returns the current date and time formatted in accordance
251  *					with datetime-format "euDateTime"
252  */
253 function noweutime() {
254 	return dateformat(new Date(),"euDateTime");
255 }
256 
257 /* ****************************************************************************
258  *	FUNCTION:		uniqueNowID
259  *	RESULT:			a unique ID String
260  *					NULL - in case of immediate uniqueness failed, caller should try later
261  *	ATTENTION:	as it is, this function only works in a single-threaded environment (like node.js),
262  *				the reason being that the access to the global prefs lastnowinc and lastnowID is
263  *				not synchronized !!
264  */
265  function uniqueNowID() {
266 	 if (!prefs.lastnowInc) {
267 		 prefs.lastnowInc = 1;
268 	 }
269 	 if (!prefs.lastnowID) {
270 		 prefs.lastnowID = nowstring()+DEC(prefs.lastnowInc++,4);
271 	 }
272 	 var curid = nowstring()+DEC(prefs.lastnowInc++,4);
273 	 if (prefs.lastnowInc>9999) prefs.lastnowInc = 1;
274 	 if (curid==prefs.lastnowID) {
275 		 return null;
276 	 } else {
277 		 prefs.lastnowID = curid;
278 		 return curid;
279 	 }
280  }
281 
282  /* ****************************************************************************
283  *	FUNCTION:		syncUniqueNowID
284  *	RESULT:			a unique ID String
285  */
286  function syncUniqueNowID() {
287 	 var id;
288 	 while (!(id=uniqueNowID()));
289 	 return id;
290  }
291  
292 /* *****************************************************************************
293  *	FUNCTION:	cutString
294  *	INPUT:		theString	-	string to be cut for readability
295  *				maxlen		-	OPTIONAL:	length to cut the string to
296  *								DEFAULT:	40 characters
297  *	RESULT:		string cut to <maxlen> characters, if necessary
298  *	DESCRIPTION:	cuts <theString> for easier log readability and returns the cut string
299  *					if it is cut, an ellipsis ("...") is appended
300  */
301 function cutString(theString,maxlen) {
302 	var mxl = (maxlen || 40);
303 	return (theString.length>mxl?theString.substring(0,mxl)+"...":theString);
304 }
305 
306 /* *****************************************************************************
307  *	FUNCTION:	repeatString
308  *	INPUT:		theString	-	string to be repeatedly copied
309  *				cnt			-	number of copies, DEFAULT: 1
310  *	DESCRIPTION:	returns a string concatenated of <cnt> copies of <theString>
311  */
312 function repeatString(theString,cnt,debug) {
313 	var cnt = cnt || 0;
314 	if (debug) logger.debug(debug+".CNT="+cnt);
315 	if (cnt<1) return '';
316 	var res = theString;
317 	while (cnt>1) {
318 		res += theString;
319 		cnt--;
320 	}
321 	return res;
322 }
323 
324 /* *****************************************************************************
325  *	FUNCTION:	fixString
326  *	INPUT:		theString	-	string to be brought to fixed length
327  *				maxlen		-	OPTIONAL:	length to bring the string to
328  *								DEFAULT:	40 characters
329  *				padchar		-	OPTIONAL:	padding character
330  *								DEFAULT:	blank (' ', UTF-codepoint 32)
331  *	RESULT:		bring string to <maxlen> characters, if necessary
332  *	DESCRIPTION:	cuts <theString> for easier log readability and returns the cut string
333  *					if it is cut, an ellipsis ("...") is appended
334  *					if shorter, pad with <padchar>
335  */
336 function fixString(theString,maxlen,padchar) {
337 	var mxl = (maxlen || 40);
338 	var pdc = (padchar || ' ');
339 	if (theString.length>mxl)
340 		return theString.substring(0,mxl)+"..."
341 	else if (theString.length<mxl) 
342 		return theString+repeatString(mxl-theString.length)
343 	else 
344 		return theString;
345 }
346 
347 /* *****************************************************************************
348  *	FUNCTION:	endsWith
349  *	INPUT:		target	-	string to be looked at
350  *				pat		-	pattern string to be found at end of <target>
351  *	RETURNS:	true of <target> ends with <pat>
352  */
353 function endsWith(target,pat) {
354 	return (target.substr(0-pat.length)==pat);
355 }
356 
357 /* *****************************************************************************
358  *	FUNCTION:		find_in_array
359  *	INPUT:			arr		-	array to find <obj> in
360  *					obj		-	object to find in <arr>
361  *	OUTPUT:			sta.loc	-	index of element in <arr> where <obj> was found
362  *	RESULT:			true of <obj> is found in <arr>
363  */
364  function find_in_array(arr,obj,sta) {
365 	 if (!arr) return false;
366 	 for (var i = 0; i<arr.length;i++) {
367 		 // logger.debug("FIA: compare "+obj+" against "+arr[i]+" at position "+i);
368 		 if (obj === arr[i]) {
369 			 if (sta) sta.loc = i;
370 			 return true;
371 		 }
372 	 }
373 	 return false;
374  }
375  
376  /* ****************************************************************************
377   *	FUNCTION:	copyObject
378   *	INPUT:		t	-	target object
379   *				o	-	any javascript data
380   *	RETURNS:	a deep copy of <o>
381   *	DESCRIPTION:	performs a deep copy of <o> and returns it
382   *				producing copies is controlled based on the type of <o>:
383   *				primitive types are automatically copied by javascript
384   *				objects are deep-copied attribute-wise
385   *					array strucure is preserved as all attributes are copied
386   *				EXCEPTION: functions are not copied, only references
387   */
388 function copyObject(t,o,dep) {
389 	var dep = dep || 100; 
390 	if (t===undefined || t===null) {
391 		throw new TypeError("Target to copyObject ("+typeof(t)+") at depth "+dep+" cannot be converted to Object");
392 	}
393 	if (o==null) return null;				// prevent handling null as object
394 	var a, no;
395 	var ot = typeof(o);
396 	if (ot == 'object') {					// o is object (including arrays)
397 		var ka = Object.keys(o);
398 		for (var ni = 0, len = ka.length; ni<len; ni++) {
399 			var nk = ka[ni];
400 			var desc = Object.getOwnPropertyDescriptor(o, nk);
401 			if (desc !== undefined && desc.enumerable) {
402 				if (!t[nk]) {
403 					if (typeof(o[nk])==='object') {			// initialize target if not yet existing ...
404 						t[nk] = {};							// ... to object ...
405 					} else {
406 						t[nk] = 0;							// ... or primitive
407 					}
408 				}
409 				t[nk] = copyObject(t[nk],o[nk],dep+1);
410 			}
411 		}
412 	} else {								// o is primitive or can be copied by reference
413 		if (typeof(t)=='object') {
414 			throw new TypeError("Target of copyObject at depth "+dep+" is object, source is primitive")
415 		} else {
416 			t = o;
417 		}
418 	}
419 	return t;
420 }
421 
422 /* ********************************************************************************
423  *	FUNCTION:		assignObject
424  *	DESCRIPTION:	backpatch/polyfill for ES6 Object.assign()
425  *					( See: https://developer.mozilla.org/de/docs/Web/JavaScript/Reference/Global_Objects/Object/assign#Polyfill )
426  */
427 function assignObject(target) {
428     'use strict';
429     if (target === undefined || target === null) {
430       throw new TypeError('Cannot convert first argument to object');
431     }
432 
433     var to = Object(target);
434     for (var i = 1; i < arguments.length; i++) {
435 		var nextSource = arguments[i];
436 		if (nextSource === undefined || nextSource === null) {
437 			continue;
438 		}
439 		nextSource = Object(nextSource);
440 
441 		var keysArray = Object.keys(nextSource);
442 		for (var nextIndex = 0, len = keysArray.length; nextIndex < len; nextIndex++) {
443 			var nextKey = keysArray[nextIndex];
444 			var desc = Object.getOwnPropertyDescriptor(nextSource, nextKey);
445 			if (desc !== undefined && desc.enumerable) {
446 				to[nextKey] = nextSource[nextKey];
447 			}
448 		}
449 	}
450 	return to;
451 }
452 
453 /* *****************************************************************************
454  * FUNCTION:	logPretty
455  * INPUT:		O 		- the Object to log to console
456  *				PRETEXT	-	prefix text, optional
457  * DESCRIPTION:	log <o> to console by using util.inspect
458  */
459 function logPretty(o,pretext) {
460 	if (pretext) {
461 		logger.debug(pretext+" ",util.inspect(o, { showHidden: true, depth: 50 }));
462 	} else {
463 		logger.debug(util.inspect(o, { showHidden: true, depth: 50 }));
464 	}
465 }
466 
467 /* *****************************************************************************
468  * FUNCTION:	logJSON
469  * INPUT:		O 		- the Object to log to console
470  *				PRETEXT	-	prefix text, optional
471  * DESCRIPTION:	log <o> to console by using JSON.stringify
472  */
473 function logJSON(o,pretext) {
474 	var str;
475 	if (!o) {
476 		str = "=<null_or_undefined>";
477 	} else if (typeof(o)=="object") {
478 		str = ":\n--BEGIN--"+JSON.stringify(JSON.decycle(o),null,2)+"\n--END----";
479 	} else {
480 		str = "=\""+o+"\"";
481 	}
482 	if (pretext) {
483 		logger.debug(pretext+str);
484 	} else {
485 		logger.debug(str);
486 	}
487 }
488 
489 /* *****************************************************************************
490  * FUNCTION:	logObject
491  * INPUT:		O 		- the Object to log to console
492  *				DESC	- a description to put before each line
493  *				PFX 	- a prefix to put in front of each line, used for indentation
494  *
495  */
496 function logObject(o,desc,pfx,dep,seen) {
497 	if (dep>40) {
498 		logger.debug(pfx+"...<max depth(40) reached, more levels suppressed>...");
499 		return;
500 	}
501 	if (!seen)	seen = new Array();
502 	if (!pfx)	pfx = "";
503 	var ts;
504 	var idx = 0;
505 	var maxidx = 0;
506 	if (util.isArray(o)) {
507 		maxidx = 10;
508 	}
509 	if (typeof o === 'object') {
510 		if (find_in_array(seen,o)) {
511 			return;
512 		}
513 		seen.push(o);
514 		logger.debug(pfx+desc);
515 		for (sp in o) {
516 			idx++;
517 			if ((maxidx>0) && (idx>maxidx)) {
518 				logger.debug(pfx+"...<max elements("+maxidx+") reached, more entries suppressed>...");
519 				break;
520 			}
521 			ts = typeof o[sp];
522 			if (ts === 'object') {
523 				logObject(o[sp],desc+": "+sp+":",pfx+"   ",dep+1,seen);
524 			} else if (ts === 'string') {
525 				logger.debug(pfx+"  "+sp+"["+ts+"]: "+cutString(o[sp],20));
526 			} else {
527 				logger.debug(pfx+"  "+sp+"["+ts+"]: "+o[sp]);
528 			}
529 		}
530 	} else {
531 		logger.debug(pfx+desc+":  "+o);
532 	}
533 }
534 
535 
536 /* *****************************************************************************
537  *
538  *	FUNCTION:	get
539  *	INPUT:		obj		-	object to get an attribute value from
540  *				prop	-	name of the attribute/property to get
541  *							may be multi-segment path name, segments separated by '.'
542  *	RETURNS:	value of the property prop (if not exists: null)
543  */
544 function get(obj, prop) {
545 	var sobj = obj;
546 	var sprop = prop;
547 	var parts = prop.split('.'),
548 		last = parts.pop();
549 
550 	while ((prop = parts.shift())) {
551 		obj = obj[prop];
552 
553 		if (obj == null) {
554 			return null;
555 		}
556 	}
557 	if (!obj) {
558 		// logger.error("GET.OBJ_IS_UNDEF.SPROP="+sprop);
559 		// logJSON(sobj,"GET.SOBJ");
560 		return null;
561 	}
562 	return obj[last];		// we have successfully traversed the tree up to the last object
563 							// if the object has the property 'last', return it, otherwise return undefined
564 }
565 
566 /* *****************************************************************************
567  *
568  *	FUNCTION:	get2
569  *	INPUT:		obj			- object to get the attribute value from as first choice
570  *				obj2		- object to get the attribute value from as second choice
571  *				prop		- path-like name of the attribute/property to get, segments separated by '.'
572  *	RESULT:		value of the property <prop>, null or undefined if not exists
573  */
574 function get2(obj,obj2,prop) {
575 	var res = get(obj,prop);
576 	if (!res) res = get(obj2,prop);
577 	return res;
578 }
579  
580 /* *****************************************************************************
581  *
582  *	FUNCTION:		noop
583  *	DESCRIPTION:	returns an empty string
584  */
585 function noop() {
586 	return '';
587 }
588 
589 
590  /* ****************************************************************************
591   *	FUNCTION:		default_param
592   * INPUT:			O		- object reference to set parameter in
593   *					NAM		- name of parameter to set
594   *					DEFVAL	- default value for parameter if not already set
595   *							  if <defval> is a function, it is called with <oparam> as single parameter
596   *					OPARAM  - optional parameter passed to <defval> if <defval> is a function
597   * DESCRIPTION:	if O does not already have NAM as attribute, set it to DEFVAL
598   *					if <defval> is a function, it is called with <oparam> as single parameter
599   *					  and null as this-object
600   */
601 function default_param(o,nam,defval,oparam) {
602 	if (!o.hasOwnProperty(nam)) {
603 		if (typeof(defval)==='function') {
604 			o[nam] = defval.call(null,oparam);
605 		} else {
606 			o[nam]=defval;
607 		}
608 	}
609 }
610 
611 /* *****************************************************************************
612  *	FUNCTION:		error_callback
613  *	INPUT:			callback	-	callback function to be invoked
614  *					err			-	optional error object to be propagated
615  *					netxt		-	message text for propagating error object
616  *					robj		-	optional result object to be propagated with error
617  *	DESCRIPTION:	initiates error callback of <that>
618  */
619 function error_callback(callback,err,netxt,robj) {
620 	var nerr;
621 	if (!netxt) netxt = "Fehler";
622 	if (err) {
623 		nerr = new Error(netxt+": "+err.message);
624 	} else {
625 		nerr = new Error(netxt);
626 	}
627 	logger.error(nerr.message);
628 	if (robj) {
629 		logger.error("-Error-Context: ",util.inspect(robj,{ showHidden: true, depth: 50}));
630 	}
631 	callback(nerr,robj);
632 	return;
633 }
634 
635  /* ****************************************************************************
636   *
637   *	FUNCTION:		eval_expression
638   *	INPUT:			exp			- expression to be processed
639   *									if <exp> starts with #, debug information is logged
640   *					ctxobj		- context object to be the context in the expression
641   *					ctxobj2		- fallback context to look for objects not found in <ctxobj>
642   *					options		- option object, options are:
643   *						dbg		-	if true, debug information is logged a debug level
644   *	RETURNS:		result of the processed expression
645   *	DESCRIPTION:	processes the expression in <exp>:
646   *					if it starts with 'jsexpr:', the rest is eval()-ed as javascript-expression 
647   *						and its return value is the result of this function
648   *					otherwise, the result is the value of the attribute EXP in the CTXOBJ
649   */
650   function eval_expression(exp,ctxobj,ctxobj2,options) {
651 	  var context = ctxobj;
652 	  var context2 = ctxobj2;
653 	  var jsexpr;
654 	  var jseanam;
655 	  var jseapath;
656 	  var expres;
657 	  var ctxbo;
658 	  var dbg = false;
659 	  if (options) {
660 		  dbg = options.dbg || dbg;
661 	  }
662 	  if (dbg) {
663 		  logJSON(exp,"EVAL_EXPRESSION.EXP");
664 	  }
665 	  if (exp) {
666 		if (exp.substr(0,1)=='#') {
667 			exp = exp.substr(1);
668 			if (dbg) logger.debug("EVAL_EXPRESSION:EXP="+exp);
669 		}
670 		if (exp.substr(0,7)=='jsexpr:') {						// the rest of EXP is javascript
671 			if (exp.charAt(7)=='@') {							// the javascript is stored in an attribute inside CTXOBJ/CTXOBJ2, the rest of EXP is the name of this attribute
672 				jseanam = exp.substr(8);						// string path attribute name
673 				jsexpr = get2(ctxobj,ctxobj2,jseanam);
674 				if (!jsexpr) jsexpr = "__NOTFOUNDJSEANAMGET2__";
675 			} else {
676 				jsexpr = exp.substr(7);							// the javascript is directly in EXP after the 'jsexpr:'
677 			}
678 			if (jsexpr.startsWith("__NOTF")) {					// something not found ...
679 				if (dbg) {
680 					logJSON(jsexpr,"EVAL_EXPRESSION.JSEXPR_NOTFOUND");
681 				}
682 				return jsexpr;									// return this to initiate the caller to try possibly with fallback context
683 			}
684 			try {
685 				if (dbg) {
686 					logger.debug("EVAL_EXPRESSION: JSEXPR=>>>"+jsexpr+"<<<");
687 				}
688 				expres = eval(jsexpr);
689 			}
690 			catch (e) {
691 				var eeerr = new Error("EVAL_EXPRESSION: Error in EVAL: "+e.message);
692 				logger.error(eeerr.message);
693 				expres = "__NOTFOUNDERRORINPARSINGEXPRESSION__";
694 			}
695 			if (dbg) {
696 				logJSON(expres,"EVAL_EXPRESSION.EXPRES_EXPRESSION");
697 			}
698 			return expres;
699 		} else {												// no javascript, EXP contains attribute name, possibly multi-level name, separated by "."
700 			expres = get2(ctxobj,ctxobj2,exp);
701 			if (dbg) {
702 				logJSON(expres,"EVAL_EXPRESSION.EXPRES_ATTRIBUTE");
703 			}
704 			expres = (expres)?expres:"__NOTFOUND__"
705 			return expres;
706 		}
707 	  } else {
708 		  return "";
709 	  }
710   }
711  
712  /* ****************************************************************************
713   *	FUNCTION:		populate
714   *	INPUT:			template	- template string containing placeholders delimited by "{{" and "}}"
715   *					data		- object with attributes named like the placeholders
716   *					adata		- optional object with attributes named like the placeholders
717   *				alternatively the function can be called with one single object parameter:
718   *					cobj		- one object with attributes:
719   *						template
720   *						rex			-	regular expression to locate placeholders, 
721   *										defaults to /\{\{([\w\.\(\)"':;\?@]*)\}\}/g
722   *										which means:
723   *											everything between "{{" and "}}" and containing: word-characters, dots, parantheses (), colon, semicolon, question-marks and at-signs
724   *											will be recognized as placeholder name and passed to eval_expression to be replaced globally (the "g" modifier at the end)
725   *						data
726   *						adata
727   *						dbg			-	output debug information if true (and logger debug is enabled), defaults to false
728   *
729   *	RETURNS:		string as template with placeholders replaced by 
730   *					the result of processing the detected placeholders as specified by <rex>, default: the string between "{{" and "}}"
731   *					context of processing is <data> and <adata> in following precedence:
732   *					first the string s processed against <data>.
733   *					If this is not successful (returns string starting with "__NOTF"),
734   *					then the string is processed against <adata>
735   *					if this is not successful (starting with "__NOTF"), an empty string is returned
736   *					if either is successful, the result of the processing by eval_expression()
737   *					is returned as result, i.e. the placeholder is replaced by the result
738   *			=> TODO: the current default implementation (for <rex>) allows no arbitrary characters in the placeholders,
739   *				therefore, javascript expressions in placeholders have to be hidden in context variables
740   *
741   *	SEE ALSO:	eval_expression()
742   *
743   */
744  function populate(template, data, adata, defval) {
745 	var rex;
746 	var dbg = false;
747 	rex = /\{\{([\w\.\(\)\#"':;\?@]*)\}\}/g
748 	if (typeof(template)==='object') {
749 		 cobj = template;
750 		 template = cobj.template;
751 		 rex = cobj.rex || /\{\{([\w\.\(\)\#"':;\?@]*)\}\}/g
752 		 data = cobj.data;
753 		 adata = cobj.adata;
754 		 dbg = cobj.dbg || false;
755 		 defval = cobj.defval || '';
756 	}
757 	defval = defval || '';
758 	if (dbg) logger.debug("POPULATE.REX=\""+rex+"\"");
759 	if (dbg) logger.debug("POPULATE.TEMPLATE=\""+template+"\"");
760 	if (!(typeof(template)==="string")) {
761 		logger.error("AUX.POPULATE.TEMPLATE=\""+template+"\".IS_NOT_STRING!!");
762 		logger.error((new Error("AUX.POPULATE.TEMPLATE=\""+template+"\".IS_NOT_STRING!!")).stack);
763 		return "<NOT_STRING>";
764 	}
765 	return template.replace(rex,
766 	                        function(m,key) {
767 								if (dbg) logger.debug("TEMPLATE.REPLACE.KEY=\""+key+"\"");
768 								if (dbg) logger.debug("TEMPLATE.REPLACE.KEY=\""+key+"\"");
769 								var res = eval_expression(key,data,adata,{dbg:dbg});
770 								return (res)?(typeof(res)==="string")?(res.startsWith("__NOTF")?defval:res):res:defval;
771 							});
772  }	
773  
774  
775  /* ****************************************************************************
776   *	FUNCTION:	populate_all_strings
777   *	INPUT:		O		-	object, whose STRING attributes shall have templates expanded
778   *				ATT		-	primary context of values for placeholders
779   *				SATT	-	secondary context of values for placeholders. If a placeholder cannot
780   *							be replaced from ATT, SATT is tried
781   *	DESCRIPTION:	all strings in O are checked for placeholders (marked by {{..}} )
782   *					marked placeholders are replaced by attributes from ATT. If ATT has no replacement,
783   *					SATT is tried
784   *					The detailed replacement is done by populate(), see also there!
785   */
786  function populate_all_strings(o,att,satt) {
787 	 for (var an in o) {
788 		 if (typeof o[an] === 'string') {
789 			 o[an] = populate(o[an],att,satt);
790 		 }
791 	 }
792  }
793 
794  /* ****************************************************************************
795   *		FUNCTION:	find_in_paramdata
796   *		INPUT:		pdata	-	array with objects. Each object must have a "name" attribute
797   *					pname	-	name of the object to be found
798   *		RESULT:		the object with name <pname>, 
799   *					null if not found (none of the objects in pdata has an attribute 'name' with value <pname>
800   */
801  
802  function find_in_paramdata(pdata,pname) {
803 	 for (var pdi = 0; pdi<pdata.length; pdi++) {
804 		 if (pdata[pdi].name=pname) {
805 			 return pdata[pdi];
806 		 }
807 	 }
808 	 return null;
809  }
810 
811  /* ****************************************************************************
812   *	FUNCTION:	findInObjectArray
813   *	INPUT:		oa		-	array of objects
814   *				nam		-	name of the object attribute to find
815   *				val		-	value of the object attribute to find
816   *	RETURNS:	reference to the first object in oa that has value <val> in attribute named <nam>
817   */
818  
819  function findInObjectArray(oa,nam,val) {
820 	 for (var i = 0; i<oa.length; i++) {
821 		 if (oa[i][nam]=val) {
822 			 return oa[i];
823 		 }
824 	 }
825 	 return null;
826  }
827 
828  
829 /* *****************************************************************************
830  *	FUNCTION:		process_context_expression
831  *	INPUT:			that		-	query execution context
832  *					expattnam	-	name of extension hook to be invoked
833  *					resattnam	-	OPTIONAL: 	name of attribute of <that> to which the 
834  *												result of the expression is stored (if any)
835  *	OUTPUT:			that	-	possible changes by executed expression
836  *	DESCRIPTION:	processes query extension hook javascript expression
837  *					it is assumed that <that> has an attribute named <expattnam>
838  *					this attribute shall contain a javascript-expression
839  *					this expression is executed
840  *					if the expression produces a return value, this value
841  *					is logged but not further used or kept
842  *					before execution (with javascript eval()-function), placeholders
843  *					delimited by {{..}} are replaced (using populate()-function)
844  *					If the expression returns a result and <resattnam> is specified,
845  *					the result is stored to the attribute <resattnam> of <that>
846  */ 
847 
848 function process_context_expression(that,procparname,resattnam) {
849 	var procexpr = '';
850 	if (that[procparname]) {
851 		procexpr = populate(''+that[procparname],that);
852 	} else if ((that.query) && (that.query[procparname])) {
853 		procexpr = populate(''+that.query[procparname],that.query,that);
854 	} else {
855 		procexpr = '';
856 		// logger.debug('process_context_expression: no JS-expression '+procparname+' defined');
857 		return;
858 	}
859 	// logger.debug("PROCESS_CONTEXT_EXPRESSION.EXP["+procparname+"(populated)]=\""+cutString(procexpr)+"\"");
860 	var procres;
861 	try {
862 		procres = eval(procexpr);
863 	}
864 	catch (e) {
865 		var nerr = new Error("Error in PROCESS_CONTEXT_EXPRESSION.EXP["+procparname+"(populated)]=\""+procexpr+"\": "+e.message);
866 		logger.error(nerr.message);
867 		procres = {error: nerr, expression: procexpr};
868 	}
869 	if (resattnam) {
870 		try {
871 			that[resattnam] = procres;
872 		} catch (e) {
873 			logger.error("Error assigning result to attribute "+resattnam+": "+e.message);
874 			logPretty(that,"Context: ");
875 		}
876 	}
877 	/*
878 	if (procres) {
879 		logger.debug("PROCESS_CONTEXT_EXPRESSION.RESULT[\""+procparname+"\"]: ",procres);
880 	} else {
881 		logger.debug("PROCESS_CONTEXT_EXPRESSION.RESULT[\""+procparname+"\"] is falsy");
882 	}
883 	*/
884 	// logger.debug("PROCESS_CONTEXT_EXPRESSION.END.RES: ",procres);
885 }
886 
887 var globalObjects = {};
888 
889 function createGlobalNamedObject(objnam, objval) {
890 	if (globalObjects[objnam]) {
891 		throw new Error("Global Object \""+objnam+"\" already exists");
892 		return;
893 	}
894 	globalObjects[objnam] = objval;
895 };
896 
897 // get a handle to a named global object
898 function getGlobalNamedObject(objnam) {
899 	if (globalObjects[objnam]) {
900 		return globalObjects[objnam];
901 	} else {
902 		throw new Error("Global Object \""+objnam+"\" not defined");
903 	}
904 }
905 
906 function Queue(quenam) {
907 	this.name = quenam;
908 	this.data = new Array();
909 	createGlobalNamedObject("QUEUE_"+this.name,this);
910 	this.put = function(obj) {
911 		this.data.push(obj);
912 		// logger.debug("QUEUE.PUT: ",this);
913 	}
914 	this.get = function() {
915 		return this.data.shift();
916 	}
917 }
918 
919 // get a queue by name
920 function getQueue(quenam) {
921 	return getGlobalNamedObject("QUEUE_"+quenam);
922 }
923 
924 // create a queue
925 function createQueue(quenam) {
926 	return new Queue(quenam);
927 }
928 
929 
930 /* ***********************************************************************************************
931  *
932  *	FUNCTION:		getFileType
933  *	INPUT:			fpath	-	full path to file
934  *	CALLBACK:		rfunc	-	called upon completion
935  *						parameters:
936  *							err		-	defined if an error occured
937  *							resobj	-	result object:
938  *								mime	-	string with the mime type in the form "<group>/<type>", if determinable, otherwise 'undef'
939  *								ext		-	file type extension. Typical extension for this type of file, derived from file content
940  *												if it cannot be derived from file content, falls back to <namext>
941  *											<mime> and <ext> are courtesy of the "file-type" extension ( https://github.com/sindresorhus/file-type )
942  *								namext	-	file extension from fpath; 
943  *												the extension is the last part of the pathname from the last '.' to the end
944  *												if path has no '.' => 'undef'
945  *								fexists	-	true if file really exists and is accessible, false otherwise
946  *								stats	-	file stats as returned by the fs.stat() system call ( https://nodejs.org/api/fs.html#fs_fs_stat_path_callback )
947  *	DESCRIPTION:	determines file type of file at <fpath>. 2 approaches are used:
948  *					1) use 'file-type' library reading the first bytes in the file and checking
949  *					2) if 'file-type' cannot determine the type, determine the file extension
950  *						in this case the 'mime' field stays 'undef'
951  *						if the extension cannot be determined (no dot(".") in the filename) then also the 'ext' field stays 'undef'
952  */
953 var getFileType = flow.define(
954  function(fpath, rfunc) {
955 	 this.fpath = fpath;
956 	 this.callback = rfunc;
957 	 this.resobj = null;
958 	 this.fst = "no information about opening file in getFileType()";
959 	 this.fexists = false;
960 	 this.stats = null;
961 	 fs.access(this.fpath,fs.R_OK,this);
962  },
963  function(err,result) {
964 	 if (err) {
965        logger.error("getFileType: file "+this.fpath+" does not exist or cannot be read from: "+err.message);
966 	   this.fexists = false;
967 	   this();
968 	 } else {
969 	   this.fexists = true;
970 	   fs.stat(this.fpath,this);
971 	 }
972  },
973  function(err,result) {
974 	 if (err) {
975 		 logger.error("getFileType: could not get stats of file "+this.fpath+": "+err.message);
976 		 this();
977 	 } else {
978 		 this.stats = result;
979 		readChunk(this.fpath, 0, 262,this);
980 	 }
981  },
982  function(err,buffer) {
983   if (err) {
984     logger.error("getFileType: error reading 262 bytes at start of file "+this.fpath+": "+err.message);
985 	this.resobj = null;
986 	fst = "could not read 262 bytes at start of file "+this.fpath+": "+err.message;
987   } else {
988 	this.fst = "got 262 bytes at start of file "+this.fpath;
989     this.resobj = fileType(buffer);				// try to determine file type from content
990   }
991   var di = this.fpath.lastIndexOf('.');	// does it have an extension (at least one dot (.))?
992   if (di<0) namext = 'undef'; else namext = this.fpath.substr(di+1); 
993   if (!this.resobj) {								// if not successful (resobj null), determine extension from file name
994     this.resobj = {mime: 'undef', ext: namext};
995   }
996   this.resobj.filactstat = this.fst;
997   this.resobj.namext = namext;
998   this.resobj.fexists = this.fexists;
999   this.resobj.stats = this.stats;
1000   this.callback(undefined,this.resobj);
1001  }
1002 );
1003 
1004 /* *****************************************************************************
1005  *	FUNCTION:	guessSheetrangeFromSheet
1006  *	INPUT:		sheet		-	an XLSX-worksheet object
1007  *	RETURNS:	a RANGE object
1008  *	DESCRIPTION:	tries to guess the cell range from the data in <sheet>
1009  *					iterates through all attributes of <sheet> whose names do
1010  *						not start with "!"
1011  *					These names are the addresses of the respective cells in A1-format
1012  *					these addresses are decoded and a running minimum and maximum 
1013  *						of column ("A" ...) and column ("1" ...) numbers is collected
1014  */
1015 function guessSheetrangeFromSheet(sheet) {
1016 	var rowmin = 0;
1017 	var rowmax = 0;
1018 	var colmin = 0;
1019 	var colmax = 0;
1020 	var xa = {};
1021 	for (ca in sheet) {
1022 		if (ca.substr(0,1)!="!") {
1023 			xa = XLSX.utils.decode_cell(ca);
1024 			if (!rowmin) rowmin = xa.r;
1025 			if (!rowmax) rowmax = xa.r;
1026 			if (!colmin) colmin = xa.c;
1027 			if (!colmax) colmax = xa.c;
1028 			if (xa.r<rowmin) rowmin = xa.r;
1029 			if (xa.r>rowmax) rowmax = xa.r;
1030 			if (xa.c<colmin) colmin = xa.c;
1031 			if (xa.c>colmax) colmax = xa.c;
1032 		}
1033 	}
1034 	return {s: {r: rowmin, c: colmin}, e: {r: rowmax, c: colmax}};
1035 }
1036  
1037 /* *****************************************************************************
1038  *	FUNCTION:	excelSheetRange
1039  *	INPUT:		worksheet	-	an XLSX.worksheet object
1040  *	RESULT:		an XLSX.range object:
1041  *					s	-	start cell coordinates
1042  *						r	-	row number
1043  *						c	-	column number
1044  *					e	-	end cell coordinates
1045  *						r	-	row number
1046  *						c 	-	column number
1047  */
1048  function excelSheetRange(worksheet) {
1049 	var sheetrange = worksheet['!range'];
1050 	if (!sheetrange) {
1051 		// no '!range' object, try '!ref' object
1052 		var reftxt = worksheet['!ref'];
1053 		if (reftxt) {
1054 			var refs = reftxt.split(':');	// !ref is start:end in Excel A1-notation, e.g. "A1:D604"
1055 			return {s: XLSX.utils.decode_cell(refs[0]), e: XLSX.utils.decode_cell(refs[1])};
1056 		} else {
1057 			// no !ref object, try enumerating cell addresses
1058 			return guessSheetrangeFromSheet(this.worksheet);
1059 		}				
1060 	} else {
1061 		return sheetrange;
1062 	}
1063  }
1064  
1065 /* *****************************************************************************
1066  *	FUNCTION:	excelTableToArrayOfArrays
1067  *	INPUT:		worksheet	-	an XLSX.worksheet object
1068  *	RESULT:		a [result/dbresult] object
1069  *	DESCRIPTION:	converts the complete worksheet into a array of arrays
1070  *					cells that are not present are filled with null
1071  */
1072  function excelTableToArrayOfArrays(worksheet) {
1073 	 var range = excelSheetRange(worksheet);
1074 	 var rows = new Array();
1075 	 var crow;
1076 	 var ca;
1077 	 for (var rc = range.s.r; rc<=range.e.r; rc++) {
1078 		var crow = new Array();
1079 		for (var cc = range.s.c; cc<=range.e.r; cc++) {
1080 			ca = XLSX.utils.encode_cell({r: rc, c: cc});
1081 			if (!worksheet[ca]) {
1082 				crow.push(null);
1083 			} else {
1084 				if (worksheet[ca].v) {
1085 					crow.push(worksheet[ca].v);
1086 				} else {
1087 					logger.debug("--Cell "+ca+"[R/C:"+rc+"/"+cc+"] has no v-Attribute: ",worksheet[ca]);
1088 					crow.push('');
1089 				}
1090 			}
1091 		}
1092 		rows.push(crow);
1093 	 }
1094 	 return rows;
1095  }
1096  
1097  /* *****************************************************************************
1098  *	FUNCTION:		absolutize_filepath
1099  *	INPUT:			p		- absolute or relative pathname
1100  *	RESULT:			file path <p> converted to absolute path
1101  *	DESCRIPTION:	if <p> is not absolute, [prefs.appbase] is prepended
1102  */
1103  function absolutize_filepath(p) {
1104 	 if (path.isAbsolute(p)) {
1105 		 return p;
1106 	 } else {
1107 		 return prefs.appbase + path.sep + p;
1108 	 }
1109  }
1110 		 
1111 			
1112 /* *****************************************************************************
1113  *		The module interface
1114  */
1115 localexportobjects = {
1116 	dateformat					:	dateformat,
1117 	DEC							:	DEC,
1118 	nowstring					:	nowstring,
1119 	noweutime					:	noweutime,
1120 	uniqueNowID					:	uniqueNowID,
1121 	syncUniqueNowID				:	syncUniqueNowID,
1122 	cutString					:	cutString,
1123 	repeatString				:	repeatString,
1124 	fixString					:	fixString,
1125 	copyObject  				:	copyObject,
1126 	logObject					:	logObject,
1127 	logPretty					:	logPretty,
1128 	logJSON						:	logJSON,
1129 	get							:	get,
1130 	noop						:	noop,
1131 	default_param				:	default_param,
1132 	error_callback				:	error_callback,
1133 	populate					:	populate,
1134 	eval_expression				:	eval_expression,
1135 	populate_all_strings		:	populate_all_strings,
1136 	escapeSingleQuoteAllStrings : 	escapeSingleQuoteAllStrings,
1137 	escapeSingleQuote 			:	escapeSingleQuote,
1138 	escapeDoubleQuoteAllStrings : 	escapeDoubleQuoteAllStrings,
1139 	escapeDoubleQuote 			:	escapeDoubleQuote,
1140 	replaceControlWhitespace	:	replaceControlWhitespace,
1141 	replaceControlWhitespaceAllStrings		:	replaceControlWhitespaceAllStrings,
1142 	find_in_paramdata 			:	find_in_paramdata,
1143 	findInObjectArray			:	findInObjectArray,
1144 	find_in_array				:	find_in_array,
1145 	process_context_expression	:	process_context_expression,
1146 	endsWith					:	endsWith,
1147 	getQueue					:	getQueue,
1148 	createQueue					:	createQueue,
1149 	createGlobalNamedObject		:	createGlobalNamedObject,
1150 	getGlobalNamedObject		:	getGlobalNamedObject,
1151 	getFileType					:	getFileType,
1152 	excelSheetRange				:	excelSheetRange,
1153 	excelTableToArrayOfArrays	:	excelTableToArrayOfArrays,
1154 	absolutize_filepath     	:   absolutize_filepath,
1155 	dateFromSerialString		:	dateFromSerialString,
1156 	getWeekNr					:	getWeekNr,
1157 	dateFromWeekNr				:	dateFromWeekNr,
1158 	objectIsEmpty				:	objectIsEmpty
1159 }
1160 
1161 module.exports = function(theLogger,theMasterName,thePrefs) {
1162 	logger = theLogger;
1163 	mastername = theMasterName;
1164 	prefs = thePrefs;
1165 	logger.debug("this is the AUX instantiation follow-up of "+__filename+", the master is "+mastername+"/"+theMasterName);
1166 	var c = 0;
1167 	var oal = "";
1168 	for (var oa in module.exports) {
1169 		oal += ((c<1)?"":",")+oa;
1170 		c++;
1171 	}
1172 	// logger.debug("module.exports has "+c+" attributes: "+oal);
1173 	return module.exports;
1174 }
1175 
1176 for (var oa in localexportobjects) {
1177 		module.exports[oa] = localexportobjects[oa];
1178 }