VerityPy 1.1
Python library for Verity data profiling, quality control, remediation
exectransform.py
Go to the documentation of this file.
1#!/usr/bin/env python
2"""
3Exec Transform
4
5Process transforms.
6"""
7
8__all__ = ['do_transform']
9__version__ = '1.0'
10__author__ = 'Geoffrey Malafsky'
11__email__ = 'gmalafsky@technikinterlytics.com'
12__date__ = '20240619'
13
14
15import math
16import random
17from ..transforms import lookup, transform, optypes
18from . import recfuncs, datefuncs, numfuncs
19
20
21def do_transform(
22 transobj: transform.Transform,
23 initial_value:str,
24 result_datatype:str="",
25 hash_fields:dict=None,
26 field_values:list=None,
27 lookup_dicts:list=None,
28 hash_lookup_dicts:dict=None,
29 current_record_index:int=0,
30 current_field_index:int=-1
31 ) -> str:
32 """
33 Do Transform
34
35 Processes a transform and uses LookUpDicts as needed. For lookups and by
36 reference functions, there must also be passed in dict of field titles (lower case) to index in list.
37 Lookups also require pre-processing the LookUp Dictionaries and passing into this function that list as well
38 as the hash dictionary of the titles.
39
40 Inputs:
41 transobj: Transform object to be processed
42 initial_value: initial value to begin with. This can be empty string.
43 result_datatype: optional datatype for final value (int,real,string,date,bool) default is string
44 hash_fields: dictionary key=field title lower case, value= array index in fields. This is required for Ref operations that use referenced field_values.
45 field_values: list of record values
46 lookup_dicts: optional list of LookUpDict objects
47 hash_lookup_dicts: dictionary of hashed lookup dict title to index position
48 current_record_index: integer index of record for which this transform is being processed. 0-based. If less than 0 it is ignored.
49 current_field_index: integer index of field in output record for which this transform is being processed. 0-based. If less than 0 it is ignored.
50
51 Return: If error start with notok: otherwise string of result value
52 """
53
54 lval: int=0
55 lval1: int=0
56 dval: float=0
57 dval1: float=0
58 dval2: float=0
59 result: str=""
60 str1:str=""
61 str2:str=""
62 str3:str=""
63 fldval:str=""
64 fldval2:str=""
65 fldval3:str=""
66 if_result_true:bool=False
67 prior_was_if:bool=False
68 stop_now:bool=False
69 ismatch:bool
70 isnotmatch:bool
71 wildf:bool=False
72 wildb:bool=False
73 isok:bool=False
74 doclean:bool=False
75 flag:bool
76 doclean:bool=False
77 param1:str
78 param2:str
79 param3:str
80 oper:str
81 nfld:int=-1
82 nfld2:int=-1
83 nfld3:int=-1
84 nidx:int=-1
85 nkeys:int=0
86 n1:int=-1
87 n2:int=-1
88 items:list=[]
89 str_list:list=[]
90 dbl_list:list=[]
91 keyand:list=[]
92 keynot:list=[]
93 lkup:lookup.LookUpDict=lookup.LookUpDict()
94 i:int=-1
95 try:
96 if transobj is None or not isinstance(transobj, transform.Transform):
97 raise ValueError("empty transform supplied")
98 if initial_value is None:
99 raise ValueError("missing initial_value")
100 if field_values is None:
101 field_values=[]
102 if hash_fields is None:
103 hash_fields={}
104 if lookup_dicts is None:
105 lookup_dicts=[]
106 if hash_lookup_dicts is None:
107 hash_lookup_dicts={}
108
109 random.seed()
110
111 result=initial_value
112 result_datatype=result_datatype.lower().strip()
113 i=-1
114 while i < len(transobj.ops):
115 dval=0
116 dval1=0
117 lval=0
118 lval1=0
119 param1=""
120 param2=""
121 param3=""
122 oper=""
123 stop_now=False
124 doclean=False
125
126 i += 1
127 if prior_was_if:
128 if if_result_true:
129 i += 1 # skip over false
130 else:
131 stop_now=True # stop after doing current Op since False IF
132
133 prior_was_if= False # reset
134 if_result_true= False
135 if i>= len(transobj.ops):
136 break
137
138 oper= transobj.ops[i].title.lower()
139 param1=transobj.ops[i].param1
140 param2=transobj.ops[i].param2
141 param3=transobj.ops[i].param3
142
143 str1= recfuncs.get_math_alias(result)
144 if len(str1)>0 and not str1.startswith("notok:"):
145 if not oper.endswith("refs"):
146 if oper.startswith("mult") or oper.startswith("div") or oper.startswith("add") or oper.startswith("subtract"):
147 doclean= param2.lower()!="false"
148 else:
149 doclean=False
150 else:
151 doclean= param3.lower()!="false"
152
153 dval= numfuncs.is_real_get(str1, "number", doclean)
154 if dval== -999999:
155 dval=0
156
157 if oper.startswith("ifstr") or oper.startswith("ifnotstr"):
158 items.clear()
159 if len(param1)>0:
160 items= param1.split("|")
161
162 if oper=="noop":
163 pass
164 elif oper=="settovalue":
165 if param1.lower() in recfuncs.char_aliases:
166 result= recfuncs.char_aliases[param1.lower()]
167 else:
168 result=param1
169 elif oper=="settoindex":
170 if len(param1)==0 or (lval := numfuncs.is_int_get(param1, "number", True)) < 0:
171 lval=0 # base index
172 lval1 = current_record_index if current_record_index>=0 else 0
173 lval += lval1
174 result = str(lval)
175 elif oper=="settoref":
176 param1=param1.lower()
177 if param1 in hash_fields:
178 nfld=hash_fields[param1]
179 if nfld<0 or nfld>= len(field_values):
180 result="-novalue-"
181 else:
182 result=recfuncs.get_math_alias(field_values[nfld])
183 else:
184 result="-novalue-"
185 elif oper=="settorandom":
186 flag=False
187 if len(param3)>0 and (dval := numfuncs.is_real_get(param3, "number", True)) != -999999:
188 if 0 <= dval <= 1:
189 if random.random()<dval:
190 flag=True
191 else:
192 flag=True
193 dval=0
194 if flag:
195 if len(param1)==0 or (dval := numfuncs.is_real_get(param1, "number", True)) == -999999:
196 dval=0
197 if len(param2)==0 or (dval2 := numfuncs.is_real_get(param2, "number", True)) == -999999:
198 dval2=0
199 if dval>=dval2:
200 dval2 += 1
201 dval1= random.random() * (dval2-dval) + dval
202 result= str(round(dval1,5))
203 else:
204 result=str(9.99e10)
205 elif oper=="settofreqlist":
206 flag=False
207 if len(param1)>0 and len(param2)>0:
208 str_list=param1.split("|")
209 dbl_list.clear()
210 flag=True
211 dval=0
212 for s in str_list:
213 if (dval1 := numfuncs.is_real_get(s, "number", True))!= -999999:
214 dval += dval1
215 dbl_list.append(dval)
216 else:
217 flag=False
218 break
219 if dval==0 or len(dbl_list)==0:
220 flag=False
221 if flag:
222 str_list=param2.split("|")
223 if len(str_list)!=len(dbl_list):
224 flag=False
225 else:
226 for j in range(len(dbl_list)):
227 dbl_list[j] /= dval
228 nidx=-1
229 if flag:
230 dval=-1
231 if len(param3)>0:
232 if (dval := numfuncs.is_real_get(param3, "number", True))!= -999999:
233 if 0 > dval > 1:
234 dval=1
235 elif param3.lower() in hash_fields:
236 nfld= hash_fields[param3.lower()]
237 str1= recfuncs.get_math_alias(field_values[nfld])
238 if len(str1)>0 and (dval := numfuncs.is_real_get(str1, "number", True))== -999999:
239 dval= -1
240 if 0 > dval > 1:
241 dval= random.random()
242 for j in range(len(dbl_list)):
243 if dval< dbl_list[j]:
244 nidx=j
245 break
246 if not flag or nidx<0:
247 result="-notassign-"
248 else:
249 result= recfuncs.get_math_alias(str_list[nidx])
250 elif oper=="round":
251 if (dval := numfuncs.is_real_get(result, "number", True))!= -999999:
252 n1=0
253 if len(param1)>0 and numfuncs.is_int(param1):
254 n1=max(0,int(param1))
255
256 result= str(round(dval,n1))
257 if result_datatype=="real" and "." not in result:
258 result += ".00"
259 elif oper=="floor":
260 if (dval := numfuncs.is_real_get(result, "number", True))!= -999999:
261 result= str(math.floor(dval))
262 if result_datatype=="real" and "." not in result:
263 result += ".00"
264 elif oper=="ceiling":
265 if (dval := numfuncs.is_real_get(result, "number", True))!= -999999:
266 result= str(math.ceil(dval))
267 if result_datatype=="real" and "." not in result:
268 result += ".00"
269 elif oper=="cleannumber":
270 result= numfuncs.clean_number(result)
271 elif oper in ["multbyref","divbyref","addbyref","subtractbyref","divfromref","subtractfromref"]:
272 param1=param1.lower()
273 doclean= param2.lower()!="false"
274 dval1=0
275 if param1 in hash_fields:
276 nfld= hash_fields[param1]
277 str1= recfuncs.get_math_alias(field_values[nfld])
278 if doclean:
279 str1= numfuncs.clean_number(str1)
280 if (dval1 := numfuncs.is_real_get(str1, "number", False))== -999999:
281 dval1= 0
282
283 if oper.startswith("mult"):
284 dval *= dval1
285 elif oper.startswith("divfrom"):
286 if dval==0:
287 dval= 9.99e10
288 else:
289 dval = dval1/dval
290 elif oper.startswith("div"):
291 if dval1==0:
292 dval= 9.99e10
293 else:
294 dval /= dval1
295 elif oper.startswith("add"):
296 dval += dval1
297 elif oper.startswith("subtractfrom"):
298 dval= dval1- dval
299 elif oper.startswith("subtract"):
300 dval -= dval1
301 if result_datatype.startswith("int"):
302 result= str(round(dval))
303 else:
304 result= str(dval)
305 elif oper in ["multrefs","addrefs"]:
306 if len(param2)>0:
307 param1 += "," + param2
308 param1= param1.lower()
309 doclean= param3.lower()!="false"
310 str_list= param1.split(",")
311 dval=0
312 for j in range(len(str_list)):
313 dval1=0
314 str1= str_list[j].lower().strip()
315 if str1 in hash_fields:
316 nfld= hash_fields[str1]
317 str1= recfuncs.get_math_alias(field_values[nfld])
318 if doclean:
319 str1= numfuncs.clean_number(str1)
320 if (dval1 := numfuncs.is_real_get(str1, "number", False))== -999999:
321 dval1= 0
322 if j==0:
323 dval= dval1
324 else:
325 if oper=="multrefs":
326 dval *= dval1
327 elif oper=="addrefs":
328 dval += dval1
329 if result_datatype.startswith("int"):
330 result= str(round(dval))
331 else:
332 result= str(dval)
333 elif oper in ["mult","div","divfrom","add","subtract","subtractfrom"]:
334 dval=0
335 dval1=0
336 doclean= param2.lower()!="false"
337 if doclean:
338 result= numfuncs.clean_number(result)
339 if (dval := numfuncs.is_real_get(result, "number", False))== -999999:
340 dval= 0
341 param1= recfuncs.get_math_alias(param1)
342 if doclean:
343 param1= numfuncs.clean_number(param1)
344 if (dval1 := numfuncs.is_real_get(param1, "number", False))== -999999:
345 dval1= 0
346 if oper=="mult":
347 dval *= dval1
348 elif oper=="div":
349 if dval1==0:
350 dval= 9.99e10
351 else:
352 dval /= dval1
353 elif oper=="divfrom":
354 if dval1==0:
355 dval= 0
356 elif dval==0:
357 dval= 9.99e10
358 else:
359 dval = dval1/dval
360 elif oper=="add":
361 dval += dval1
362 elif oper=="subtract":
363 dval -= dval1
364 elif oper=="subtractfrom":
365 dval= dval1-dval
366 if result_datatype.startswith("int"):
367 result= str(round(dval))
368 else:
369 result= str(dval)
370 elif oper in ["abs","negate"]:
371 if result_datatype.startswith("int"):
372 lval=0
373 if (lval := numfuncs.is_int_get(result, "number", True))== -999999:
374 lval= 0
375 if (dval := numfuncs.is_real_get(result, "number", True))!= -999999:
376 lval= math.floor(dval)
377
378 if oper=="abs":
379 lval= abs(lval)
380 elif oper=="negate":
381 lval *= -1
382 result= str(lval)
383 else:
384 if (dval := numfuncs.is_real_get(result, "number", True))== -999999:
385 dval=0
386 if oper=="abs":
387 dval= abs(dval)
388 elif oper=="negate":
389 dval *= -1
390 result= str(dval)
391 elif oper in ["log","ln"]:
392 dval=0
393 if (dval := numfuncs.is_real_get(result, "number", True))== -999999:
394 dval=0
395 if dval>0:
396 if oper=="log":
397 dval1= math.log10(dval)
398 elif oper=="ln":
399 dval1= math.log(dval)
400 else:
401 dval1= -10e6
402 if result_datatype.startswith("int"):
403 lval= math.floor(dval1)
404 result= str(lval)
405 else:
406 result= str(dval1)
407 elif oper in ["pow10","powe"]:
408 dval=0
409 if (dval := numfuncs.is_real_get(result, "number", True))== -999999:
410 dval=0
411 if dval==0:
412 result="1"
413 else:
414 if oper=="pow10":
415 dval= pow(10,dval)
416 else:
417 dval= math.exp(dval)
418 result= str(dval)
419 elif oper=="setdecimal":
420 n1=result.find(".")
421 if n1==0:
422 str1=result[1:]
423 result=""
424 elif n1>0:
425 str1=result[n1+1:]
426 result=result[:n1]
427 else:
428 str1=""
429 n1=-1
430 if numfuncs.is_int(param1):
431 n1=int(param1)
432 n1= max(n1,0)
433 if n1>0:
434 n2=len(str1)
435 if n1>n2:
436 for j in range(n1-n2):
437 str1 += "0"
438 elif n1<n2:
439 str1=str1[:n1]
440 result += "." + str1
441 elif oper=="trim":
442 result=result.strip()
443 elif oper=="ltrim":
444 result=result.lstrip()
445 elif oper=="rtrim":
446 result=result.rstrip()
447 elif oper=="tolower":
448 result=result.lower()
449 elif oper=="toupper":
450 result=result.upper()
451 elif oper=="totitle":
452 result=result.capwords()
453 elif oper in ["front","end","back","before","after"]:
454 if len(param1)>0:
455 str1= recfuncs.convert_char_aliases(param1).lower()
456 n1= result.lower().find(str1)
457 if n1>=0:
458 n2= len(str1)
459 if oper=="front":
460 result= result[:(n1+n2)]
461 elif oper in ("end","back"):
462 result=result[n1:]
463 elif oper=="before":
464 result=result[:n1]
465 elif oper=="after":
466 result=result[(n1+n2):]
467 elif oper in ("frontn","endn", "backn"):
468 if numfuncs.is_int(param1):
469 n1=int(param1)
470 if 0< n1 < len(result):
471 if oper=="frontn":
472 result= result[:n1]
473 elif oper in ("endn","backn"):
474 result=result[(len(result)-n1):]
475 elif oper=="mid":
476 str1= recfuncs.convert_char_aliases(param1)
477 str2= recfuncs.convert_char_aliases(param2)
478 n1= result.lower().find(str1.lower())
479 if 0< n1 < len(result):
480 result=result[n1:]
481 if len(str2)>0:
482 n1= result.lower().find(str2.lower())
483 if 0< n1 < len(result):
484 result=result[:n1]
485 elif oper=="midn":
486 if numfuncs.is_int(param1):
487 n1=int(param1)
488 if 0< n1 < len(result):
489 result=result[n1:]
490 if numfuncs.is_int(param2):
491 n2=int(param2)
492 if 0< n2 < len(result):
493 result=result[:n2]
494 elif oper=="charat":
495 if numfuncs.is_int(param1):
496 n1=int(param1)
497 if 0< n1 < len(result):
498 result=result[n1:n1+1]
499 elif oper=="setlength":
500 n1=-1
501 if numfuncs.is_int(param1):
502 n1=int(param1)
503 param2=param2.lower()
504 if n1>0:
505 if "left" in param2 or "front" in param2:
506 str1="left"
507 else:
508 str1="right"
509 str2= recfuncs.convert_char_aliases(param3)
510 if len(str2)==0:
511 str2="x"
512 elif len(str2)>1:
513 str2=str2[:1]
514 if n1< len(result):
515 if str1=="left":
516 result= result[(len(result)-n1):]
517 else:
518 result=result[:n1]
519 elif n1>len(result):
520 n2=n1-len(result)
521 for j in range(n2):
522 if str1=="left":
523 result = str2 + result
524 else:
525 result += str2
526 elif oper=="prepend":
527 str1= recfuncs.convert_char_aliases(param1)
528 result = str1 + result
529 elif oper=="append":
530 str1= recfuncs.convert_char_aliases(param1)
531 result += str1
532 elif oper=="remove":
533 str2= recfuncs.convert_char_aliases(param1).lower()
534 str1=result.lower()
535 n1= str1.find(str2)
536 while n1>=0:
537 if n1==0:
538 result=result[len(str2):]
539 elif n1==len(result)-len(str2):
540 result=result[:-len(str2)]
541 else:
542 str1=result[:n1]
543 result=result[(n1+len(str2)):]
544 result= str1 + result
545 str1=result.lower()
546 n1= str1.find(str2)
547 elif oper=="replace":
548 str2= recfuncs.convert_char_aliases(param1).lower()
549 str3= recfuncs.convert_char_aliases(param2)
550 if str2 != str3.lower():
551 str1=result.lower()
552 n1= str1.find(str2)
553 while n1>=0:
554 if n1==0:
555 result=result[len(str2):] + str3
556 elif n1==len(result)-len(str2):
557 result=result[:-len(str2)] + str3
558 else:
559 str1=result[:n1]
560 result=result[(n1+len(str2)):]
561 result= str1 + str3 + result
562 str1=result.lower()
563 n1= str1.find(str2)
564 elif oper=="exceldatenumbertoiso":
565 result= datefuncs.convert_excel_date_to_iso(result)
566 elif oper=="datetoiso":
567 result= datefuncs.convert_date_to_iso(result, param1)
568 elif oper=="settoisodate":
569 param1=param1.lower().strip()
570 flag=False
571 if param1 in ["today","now",""]:
572 if param1=="now":
573 flag=True
574 result= datefuncs.get_current_iso_datetime(flag, param2)
575 else:
576 if "t" in param1:
577 flag=True
578 str1= datefuncs.is_iso_date_str(param1,flag)
579 if str1.startswith("true"):
580 if ":" in str1:
581 result=str1[str1.find(":")+1:]
582 else:
583 result=param1
584 else:
585 result=""
586 elif oper=="convertmainframenumber":
587 result= numfuncs.convert_mainframe(result)
588 elif oper=="convertfromexp":
589 result= numfuncs.get_value_from_suspect_exp(result)
590 elif oper.startswith(("ifgte","ifgt","iflte","iflt","ifnotgte","ifnotgt","ifnotlte","ifnotlt","ifeq","ifnoteq")):
591 # includes byref
592 prior_was_if=True
593 if_result_true=False
594 param1=param1.lower()
595 dval1=0
596 if oper.endswith("ref"):
597 if len(param1)==0:
598 raise ValueError("missing reference field title in: " + oper)
599 if param1 in hash_fields:
600 nfld= hash_fields[param1]
601 str1= recfuncs.get_math_alias(field_values[nfld])
602 if (dval1 := numfuncs.is_real_get(str1, "number", True))== -999999:
603 dval1=0
604 else:
605 raise ValueError("reference field title not in fields: " + param1 + ", oper=" + oper)
606 oper=oper[:-3]
607 else:
608 str1= recfuncs.get_math_alias(param1)
609 if (dval1 := numfuncs.is_real_get(str1, "number", True))== -999999:
610 dval1=0
611 # dval set at start of next index in Ops loop
612 if oper in ["ifgte","ifnotgte"]:
613 if dval >= dval1:
614 if_result_true=True
615 elif oper in ["ifgt","ifnotgt"]:
616 if dval > dval1:
617 if_result_true=True
618 elif oper in ["iflte","ifnotlte"]:
619 if dval <= dval1:
620 if_result_true=True
621 elif oper in ["iflt","ifnotlt"]:
622 if dval < dval1:
623 if_result_true=True
624 elif oper in ["ifeq","ifnoteq"]:
625 if dval == dval1:
626 if_result_true=True
627 if oper.startswith("ifnot"):
628 if_result_true = not if_result_true
629 if i== len(transobj.ops)-1:
630 result= str(if_result_true).lower()
631 elif oper in ["ifempty","ifnotempty"]:
632 prior_was_if=True
633 if_result_true=False
634 if len(result)==0:
635 if_result_true=True
636 if oper.startswith("ifnot"):
637 if_result_true= not if_result_true
638 if i== len(transobj.ops)-1:
639 result= str(if_result_true).lower()
640 elif oper in ["ifstreq","ifnotstreq"]:
641 prior_was_if=True
642 if_result_true=False
643 str1=result
644 if not param2=="true":
645 str1=str1.lower()
646 for j in range(len(items)):
647 str2= recfuncs.convert_special_notation(items[j])
648 if not param2=="true":
649 str2=str2.lower()
650 if str1==str2:
651 if_result_true=True
652 break
653 if oper.startswith("ifnot"):
654 if_result_true= not if_result_true
655 if i== len(transobj.ops)-1:
656 result= str(if_result_true).lower()
657 elif oper in ["ifstrstarts","ifnotstrstarts"]:
658 prior_was_if=True
659 if_result_true=False
660 str1=result
661 if not param2=="true":
662 str1=str1.lower()
663 for j in range(len(items)):
664 str2= recfuncs.convert_special_notation(items[j])
665 if not param2=="true":
666 str2=str2.lower()
667 if str1.startswith(str2):
668 if_result_true=True
669 break
670 if oper.startswith("ifnot"):
671 if_result_true= not if_result_true
672 if i== len(transobj.ops)-1:
673 result= str(if_result_true).lower()
674 elif oper in ["ifstrends","ifnotstrends"]:
675 prior_was_if=True
676 if_result_true=False
677 str1=result
678 if not param2=="true":
679 str1=str1.lower()
680 for j in range(len(items)):
681 str2= recfuncs.convert_special_notation(items[j])
682 if not param2=="true":
683 str2=str2.lower()
684 if str1.endswith(str2):
685 if_result_true=True
686 break
687 if oper.startswith("ifnot"):
688 if_result_true= not if_result_true
689 if i== len(transobj.ops)-1:
690 result= str(if_result_true).lower()
691 elif oper in ["ifstrcontains","ifnotstrcontains"]:
692 prior_was_if=True
693 if_result_true=False
694 str1=result
695 if not param2=="true":
696 str1=str1.lower()
697 if param3=="true":
698 str1=str1.replace(" ","")
699 for j in range(len(items)):
700 str2= recfuncs.convert_special_notation(items[j])
701 if not param2=="true":
702 str2=str2.lower()
703 if str2 in str1:
704 if_result_true=True
705 break
706 if oper.startswith("ifnot"):
707 if_result_true= not if_result_true
708 if i== len(transobj.ops)-1:
709 result= str(if_result_true).lower()
710 elif oper in ["ifint","ifnotint"]:
711 prior_was_if=True
712 if_result_true=False
713 lval=0
714 if numfuncs.is_int(result):
715 lval=int(result)
716 if_result_true=True
717 if param1=="positive" and lval<=0:
718 if_result_true=False
719 elif param1=="negative" and lval>=0:
720 if_result_true=False
721 if oper.startswith("ifnot"):
722 if_result_true= not if_result_true
723 if i== len(transobj.ops)-1:
724 result= str(if_result_true).lower()
725 elif oper in ["ifreal","ifnotreal"]:
726 prior_was_if=True
727 if_result_true=False
728 dval=0
729 if (dval := numfuncs.is_real_get(result, "number", True))== -999999:
730 dval=0
731 else:
732 if_result_true=True
733 if param1=="positive" and dval<=0:
734 if_result_true=False
735 elif param1=="negative" and dval>=0:
736 if_result_true=False
737 if oper.startswith("ifnot"):
738 if_result_true= not if_result_true
739 if i== len(transobj.ops)-1:
740 result= str(if_result_true).lower()
741 elif oper.startswith(("ifmatch","ifnotmatch")):
742 # includes byref
743 prior_was_if=True
744 if_result_true=False
745 str1=result.lower()
746 str2=""
747 param1=param1.lower()
748 if oper.endswith("ref"):
749 if len(param1)==0:
750 raise ValueError("missing reference field title in: " + oper)
751 if param1 in hash_fields:
752 nfld= hash_fields[param1]
753 str2= recfuncs.get_math_alias(field_values[nfld])
754 else:
755 raise ValueError("reference field title not in fields: " + param1 + ", oper=" + oper)
756 else:
757 str2=param1
758 str1=recfuncs.convert_special_notation(str1)
759 str2=recfuncs.convert_special_notation(str2)
760 wildf=False
761 wildb=False
762 ismatch=False
763 if str2.startswith("*"):
764 wildf=True
765 str2=str2[1:]
766 if len(str2)==0:
767 wildb=True
768 elif str2.endswith("*"):
769 wildb=True
770 str2=str2[:-1]
771 if wildf and wildb and len(str2)==0:
772 ismatch=True
773 elif len(str1)>0 and len(str2)>0:
774 if wildf and wildb and str2 in str1:
775 ismatch=True
776 elif wildf and not wildb and str1.endswith(str2):
777 ismatch=True
778 elif not wildf and wildb and str1.startswith(str2):
779 ismatch=True
780 elif not wildf and not wildb and str1==str2:
781 ismatch=True
782 if_result_true=ismatch
783 if oper.startswith("ifnot"):
784 if_result_true= not if_result_true
785 if i== len(transobj.ops)-1:
786 result= str(if_result_true).lower()
787 elif oper in ["ifisodate","ifnotisodate"]:
788 prior_was_if=True
789 if_result_true=False
790 str1= datefuncs.is_iso_date_str(result, False)
791 if_result_true= str1.startswith("true")
792 if oper.startswith("ifnot"):
793 if_result_true= not if_result_true
794 if i== len(transobj.ops)-1:
795 result= str(if_result_true).lower()
796 elif oper in ["ifdateformat","ifnotdateformat"]:
797 prior_was_if=True
798 if_result_true= datefuncs.is_date_format(result, param1)
799 if oper.startswith("ifnot"):
800 if_result_true= not if_result_true
801 if i== len(transobj.ops)-1:
802 result= str(if_result_true).lower()
803 elif oper=="lookup":
804 isok=False
805 if hash_fields is None or len(hash_fields)==0:
806 raise ValueError("no hash fields defined so cannot do transform:" + transobj.title + ",op index=" + str(i))
807 if lookup_dicts is None or len(lookup_dicts)==0:
808 raise ValueError("no lookupdicts defined, transform:" + transobj.title + ",op index=" + str(i))
809 if hash_lookup_dicts is None:
810 raise ValueError("no hash lookupdicts defined, transform:" + transobj.title + ",op index=" + str(i))
811 param1=param1.lower()
812 if param1 not in hash_lookup_dicts:
813 raise ValueError("transform lookupdict not defined:" + param1 + ",transform="\
814 + transobj.title + ",op index=" + str(i))
815 nidx= hash_lookup_dicts[param1]
816 lkup= lookup_dicts[nidx]
817 nkeys=1 # number keys defined in Op always has base value as first
818 str2=""
819 str3=""
820 nfld2=-1
821 nfld3=-1
822 fldval2=""
823 fldval3=""
824 param2=param2.lower()
825 if len(param2)>0:
826 n1= param2.find("|")
827 if "|" in param2 and len(param2)>(1+ n1):
828 nkeys += 2
829 str2= param2[:n1]
830 str3=param2[(n1+1):]
831 else:
832 nkeys += 1
833 str2=param2
834 if len(str2)>0:
835 if str2 in hash_fields:
836 nfld2=hash_fields[str2]
837 else:
838 raise ValueError("lookup field 2 is not in supplied record fields: " + str2)
839 if len(str3)>0:
840 if str3 in hash_fields:
841 nfld3=hash_fields[str3]
842 else:
843 raise ValueError("lookup field 3 is not in supplied record fields: " + str3)
844
845 if nfld2>=0:
846 fldval2= field_values[nfld2]
847 if nfld3>=0:
848 fldval3= field_values[nfld3]
849 if not lkup.is_case_sens:
850 if nfld2>=0:
851 fldval2=fldval2.lower()
852 if nfld3>=0:
853 fldval3=fldval3.lower()
854
855 for j in range(len(lkup.recs)):
856 if isok:
857 break
858
859 ismatch=False
860 for f in range(nkeys):
861 if f==0:
862 fldval= recfuncs.convert_special_notation(result)
863 if not lkup.is_case_sens:
864 fldval=fldval.lower()
865 keyand=lkup.recs[j].key1_and
866 keynot=lkup.recs[j].key1_not
867 elif f==1:
868 fldval=fldval2
869 keyand=lkup.recs[j].key2_and
870 keynot=lkup.recs[j].key2_not
871 elif f==2:
872 fldval=fldval3
873 keyand=lkup.recs[j].key3_and
874 keynot=lkup.recs[j].key3_not
875 else:
876 break
877
878 isnotmatch=False
879 # check NOT first. Done as OR so any match means NOT met
880 for k in range(len(keynot)):
881 flag=False
882 str1=keynot[k]
883 if not lkup.is_case_sens:
884 str1=str1.lower()
885
886 if f==0:
887 wildf=lkup.recs[j].key1_not_front_wild[k]
888 wildb=lkup.recs[j].key1_not_back_wild[k]
889 elif f==1:
890 wildf=lkup.recs[j].key2_not_front_wild[k]
891 wildb=lkup.recs[j].key2_not_back_wild[k]
892 elif f==2:
893 wildf=lkup.recs[j].key3_not_front_wild[k]
894 wildb=lkup.recs[j].key3_not_back_wild[k]
895
896 if wildf and wildb and len(str1)==0:
897 flag=True
898 elif len(fldval)>0 and len(str1)>0:
899 if wildf and wildb and str1 in fldval:
900 flag=True
901 elif wildf and not wildb and fldval.endswith(str1):
902 flag=True
903 elif not wildf and wildb and fldval.startswith(str1):
904 flag=True
905 elif not wildf and not wildb and fldval==str1:
906 flag=True
907 if flag:
908 isnotmatch=True
909 break
910
911 if len(keyand)==0:
912 ismatch=True
913 if not isnotmatch:
914 flag=True
915 for k in range(len(keyand)):
916 flag=False
917 str1=keyand[k]
918 if not lkup.is_case_sens:
919 str1=str1.lower()
920 if f==0:
921 wildf=lkup.recs[j].key1_and_front_wild[k]
922 wildb=lkup.recs[j].key1_and_back_wild[k]
923 elif f==1:
924 wildf=lkup.recs[j].key2_and_front_wild[k]
925 wildb=lkup.recs[j].key2_and_back_wild[k]
926 elif f==2:
927 wildf=lkup.recs[j].key3_and_front_wild[k]
928 wildb=lkup.recs[j].key3_and_back_wild[k]
929
930 if wildf and wildb and len(str1)==0:
931 flag=True
932 elif len(fldval)>0 and len(str1)>0:
933 if wildf and wildb and str1 in fldval:
934 flag=True
935 elif wildf and not wildb and fldval.endswith(str1):
936 flag=True
937 elif not wildf and wildb and fldval.startswith(str1):
938 flag=True
939 elif not wildf and not wildb and fldval==str1:
940 flag=True
941 if not flag:
942 ismatch=False
943 break
944 ismatch=True
945
946 if not ismatch or isnotmatch:
947 # failed conditions so no need to check other fields
948 break
949
950 #all fields checked so must be ok at this point to set result
951 if ismatch and not isnotmatch:
952 result= lkup.recs[j].result
953 isok=True
954 break
955
956 if stop_now:
957 break
958
959 except (RuntimeError,ValueError,OSError) as err:
960 result="notok:" + str(err)
961 return result
VerityPy.processing.exectransform.do_transform
str do_transform(transform.Transform transobj, str initial_value, str result_datatype="", dict hash_fields=None, list field_values=None, list lookup_dicts=None, dict hash_lookup_dicts=None, int current_record_index=0, int current_field_index=-1)
Definition exectransform.py:31
Generated on Sun Aug 11 2024 21:35:50 for VerityPy by doxygen 1.10.0