From 9d6f87c87afe3b01904c7522aca750d9381eefa6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 30 Dec 2023 22:43:02 +0000 Subject: [PATCH 1/5] chore: updating all.sas From 3294767c1bef99fb5f1c1f583276f949588b100d Mon Sep 17 00:00:00 2001 From: zver Date: Sun, 31 Dec 2023 00:07:02 +0000 Subject: [PATCH 2/5] feat: enabling variable names for numeric fields. #368 --- base/mp_filtercheck.sas | 83 ++++++++++++++++++++++++++---- tests/base/mp_filtercheck.test.sas | 27 +++++++++- 2 files changed, 98 insertions(+), 12 deletions(-) diff --git a/base/mp_filtercheck.sas b/base/mp_filtercheck.sas index 75885c49..16283e94 100644 --- a/base/mp_filtercheck.sas +++ b/base/mp_filtercheck.sas @@ -86,15 +86,14 @@ /** * Sanitise the values based on valid value lists, then strip out * quotes, commas, periods and spaces. - * Only numeric values should remain */ %local reason_cd nobs; %let nobs=0; data &outds; /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;*/ - set &inds; - length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000; + set &inds end=last; + length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000; drop tmp; /* quick check to ensure column exists */ @@ -110,7 +109,8 @@ data &outds; end; /* need to open the dataset to get the column type */ - dsid=open("&targetds","i"); + retain dsid; + if _n_=1 then dsid=open("&targetds","i"); if dsid>0 then do; vnum=varnum(dsid,VARIABLE_NM); if vnum<1 then do; @@ -120,11 +120,19 @@ data &outds; call symputx('reason_cd',reason_cd,'l'); call symputx('nobs',_n_,'l'); output; - return; + goto endstep; end; /* now we can get the type */ else vtype=vartype(dsid,vnum); end; + else do; + REASON_CD=cats("Could not open &targetds"); + putlog REASON_CD= dsid=; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + stop; + end; /* closed list checks */ if GROUP_LOGIC not in ('AND','OR') then do; @@ -159,15 +167,40 @@ data &outds; end; /* special missing logic */ - if vtype='N' - and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') - and cats(upcase(raw_value)) in ( + if vtype='N' & OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') then do; + if cats(upcase(raw_value)) in ( '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N' '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._' ) - then do; - /* valid numeric - exit data step loop */ - return; + then do; + /* valid numeric - exit data step loop */ + return; + end; + else if subpad(upcase(raw_value),1,1) in ( + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' + 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' + ) + then do; + /* check if the raw_value contains a valid variable NAME */ + vnum=varnum(dsid,subpad(raw_value,1,32)); + if vnum>0 then do; + /* now we can get the type */ + vtype2=vartype(dsid,vnum); + /* check type matches */ + if vtype2=vtype then do; + /* valid target var - exit loop */ + return; + end; + else do; + REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); + putlog REASON_CD= dsid=; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + goto endstep; + end; + end; + end; end; /* special logic */ @@ -189,6 +222,32 @@ data &outds; if vtype='N' then do i=1 to countc(raw_value1, ',')+1; tmp=scan(raw_value1,i,','); if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do; + if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' + 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' + ) + then do; + /* check if the raw_value contains a valid variable NAME */ + /* is not valid syntax for IN or NOT IN */ + vnum=varnum(dsid,subpad(tmp,1,32)); + if vnum>0 then do; + /* now we can get the type */ + vtype2=vartype(dsid,vnum); + /* check type matches */ + if vtype2=vtype then do; + /* valid target var - exit loop */ + return; + end; + else do; + REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); + putlog REASON_CD= dsid=; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + goto endstep; + end; + end; + end; REASON_CD='Non Numeric value provided'; putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; call symputx('reason_cd',reason_cd,'l'); @@ -221,6 +280,8 @@ data &outds; output; end; + endstep: + if last then rc=close(dsid); run; diff --git a/tests/base/mp_filtercheck.test.sas b/tests/base/mp_filtercheck.test.sas index b948cb41..66415730 100644 --- a/tests/base/mp_filtercheck.test.sas +++ b/tests/base/mp_filtercheck.test.sas @@ -53,7 +53,9 @@ AND,AND,1,age,=,.A AND,AND,1,height,<,.B AND,AND,1,age,IN,"(.a,.b,.)" AND,AND,1,age,IN,"(.A)" - +AND,AND,1,AGE,=,AGE +AND,AND,1,AGE,<,Weight +AND,AND,1,AGE,BETWEEN,"HEIGHT AND WEIGHT" ;;;; run; @@ -204,3 +206,26 @@ run; outds=work.test_results ) %let syscc=0; + + +/* invalid IN value (cannot use var names) */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,AGE,NOT IN,"(height, age)" +;;;; +run; + +%mp_filtercheck(work.inds, + targetds=work.class, + outds=work.badrecords, + abort=NO +) +%let syscc=0; +%mp_assertdsobs(work.badrecords, + desc=Invalid IN syntax, + test=HASOBS, + outds=work.test_results +) From ee5688f97fb779275f6b2deb7c9a90d5f04a4f3f Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 31 Dec 2023 00:08:50 +0000 Subject: [PATCH 3/5] chore: updating all.sas --- all.sas | 83 +++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 72 insertions(+), 11 deletions(-) diff --git a/all.sas b/all.sas index dd0cc757..c2bad28f 100644 --- a/all.sas +++ b/all.sas @@ -6377,15 +6377,14 @@ drop table &ds1, &ds2; /** * Sanitise the values based on valid value lists, then strip out * quotes, commas, periods and spaces. - * Only numeric values should remain */ %local reason_cd nobs; %let nobs=0; data &outds; /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;*/ - set &inds; - length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000; + set &inds end=last; + length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000; drop tmp; /* quick check to ensure column exists */ @@ -6401,7 +6400,8 @@ data &outds; end; /* need to open the dataset to get the column type */ - dsid=open("&targetds","i"); + retain dsid; + if _n_=1 then dsid=open("&targetds","i"); if dsid>0 then do; vnum=varnum(dsid,VARIABLE_NM); if vnum<1 then do; @@ -6411,11 +6411,19 @@ data &outds; call symputx('reason_cd',reason_cd,'l'); call symputx('nobs',_n_,'l'); output; - return; + goto endstep; end; /* now we can get the type */ else vtype=vartype(dsid,vnum); end; + else do; + REASON_CD=cats("Could not open &targetds"); + putlog REASON_CD= dsid=; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + stop; + end; /* closed list checks */ if GROUP_LOGIC not in ('AND','OR') then do; @@ -6450,15 +6458,40 @@ data &outds; end; /* special missing logic */ - if vtype='N' - and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') - and cats(upcase(raw_value)) in ( + if vtype='N' & OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') then do; + if cats(upcase(raw_value)) in ( '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N' '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._' ) - then do; - /* valid numeric - exit data step loop */ - return; + then do; + /* valid numeric - exit data step loop */ + return; + end; + else if subpad(upcase(raw_value),1,1) in ( + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' + 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' + ) + then do; + /* check if the raw_value contains a valid variable NAME */ + vnum=varnum(dsid,subpad(raw_value,1,32)); + if vnum>0 then do; + /* now we can get the type */ + vtype2=vartype(dsid,vnum); + /* check type matches */ + if vtype2=vtype then do; + /* valid target var - exit loop */ + return; + end; + else do; + REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); + putlog REASON_CD= dsid=; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + goto endstep; + end; + end; + end; end; /* special logic */ @@ -6480,6 +6513,32 @@ data &outds; if vtype='N' then do i=1 to countc(raw_value1, ',')+1; tmp=scan(raw_value1,i,','); if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do; + if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' + 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' + ) + then do; + /* check if the raw_value contains a valid variable NAME */ + /* is not valid syntax for IN or NOT IN */ + vnum=varnum(dsid,subpad(tmp,1,32)); + if vnum>0 then do; + /* now we can get the type */ + vtype2=vartype(dsid,vnum); + /* check type matches */ + if vtype2=vtype then do; + /* valid target var - exit loop */ + return; + end; + else do; + REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); + putlog REASON_CD= dsid=; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + goto endstep; + end; + end; + end; REASON_CD='Non Numeric value provided'; putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; call symputx('reason_cd',reason_cd,'l'); @@ -6512,6 +6571,8 @@ data &outds; output; end; + endstep: + if last then rc=close(dsid); run; From 532e0d535a384e1e83020d52edf83589c751b209 Mon Sep 17 00:00:00 2001 From: zver Date: Sun, 31 Dec 2023 00:35:11 +0000 Subject: [PATCH 4/5] fix: tests for char support, #368 --- base/mp_filtercheck.sas | 28 +++++++++++++++++++++++++++- tests/base/mp_filtercheck.test.sas | 1 + 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/base/mp_filtercheck.sas b/base/mp_filtercheck.sas index 16283e94..4de711c3 100644 --- a/base/mp_filtercheck.sas +++ b/base/mp_filtercheck.sas @@ -272,9 +272,35 @@ data &outds; /* output records that contain values other than digits and spaces */ if notdigit(compress(raw_value3,' '))>0 then do; + if vtype='C' and subpad(upcase(raw_value),1,1) in ( + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' + 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' + ) + then do; + /* check if the raw_value contains a valid variable NAME */ + vnum=varnum(dsid,subpad(raw_value,1,32)); + if vnum>0 then do; + /* now we can get the type */ + vtype2=vartype(dsid,vnum); + /* check type matches */ + if vtype2=vtype then do; + /* valid target var - exit loop */ + return; + end; + else do; + REASON_CD=cats("Compared Char Type (",vtype2,") is not (",vtype,")"); + putlog REASON_CD= dsid=; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + goto endstep; + end; + end; + end; + putlog raw_value3= $hex32.; REASON_CD=cats('Invalid RAW_VALUE:',raw_value); - putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=; + putlog (_all_)(=); call symputx('reason_cd',reason_cd,'l'); call symputx('nobs',_n_,'l'); output; diff --git a/tests/base/mp_filtercheck.test.sas b/tests/base/mp_filtercheck.test.sas index 66415730..c895f337 100644 --- a/tests/base/mp_filtercheck.test.sas +++ b/tests/base/mp_filtercheck.test.sas @@ -56,6 +56,7 @@ AND,AND,1,age,IN,"(.A)" AND,AND,1,AGE,=,AGE AND,AND,1,AGE,<,Weight AND,AND,1,AGE,BETWEEN,"HEIGHT AND WEIGHT" +AND,OR,2,Name,=,name ;;;; run; From c874b31b6372a95d79e0e93f4fd4ea40fce9ac64 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 31 Dec 2023 00:35:46 +0000 Subject: [PATCH 5/5] chore: updating all.sas --- all.sas | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/all.sas b/all.sas index c2bad28f..87abdc53 100644 --- a/all.sas +++ b/all.sas @@ -6563,9 +6563,35 @@ data &outds; /* output records that contain values other than digits and spaces */ if notdigit(compress(raw_value3,' '))>0 then do; + if vtype='C' and subpad(upcase(raw_value),1,1) in ( + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' + 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' + ) + then do; + /* check if the raw_value contains a valid variable NAME */ + vnum=varnum(dsid,subpad(raw_value,1,32)); + if vnum>0 then do; + /* now we can get the type */ + vtype2=vartype(dsid,vnum); + /* check type matches */ + if vtype2=vtype then do; + /* valid target var - exit loop */ + return; + end; + else do; + REASON_CD=cats("Compared Char Type (",vtype2,") is not (",vtype,")"); + putlog REASON_CD= dsid=; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + goto endstep; + end; + end; + end; + putlog raw_value3= $hex32.; REASON_CD=cats('Invalid RAW_VALUE:',raw_value); - putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=; + putlog (_all_)(=); call symputx('reason_cd',reason_cd,'l'); call symputx('nobs',_n_,'l'); output;