Skip to content

Commit

Permalink
fix: handling consecutive add+delete in mp_stripdiffs
Browse files Browse the repository at this point in the history
  • Loading branch information
^ committed Apr 30, 2024
1 parent e6da373 commit 22f0cb6
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 46 deletions.
64 changes: 41 additions & 23 deletions all.sas
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13415,7 +13415,9 @@ run;
<h4> SAS Macros </h4>
@li mf_getuniquefileref.sas
@li mf_getuniquename.sas
@li mf_getvarlist.sas
@li mf_islibds.sas
@li mf_wordsinstr1butnotstr2.sas
@li mp_abort.sas

<h4> Related Macros </h4>
Expand Down Expand Up @@ -13482,38 +13484,36 @@ create table &ds1 (drop=libref dsn) as

/* extract key values only */
%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
%local keyhash processed;
%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
create table &ds2 as
select distinct key_hash,
select key_hash as &keyhash,
tgtvar_nm,
tgtvar_type,
coalescec(oldval_char,newval_char) as charval,
coalesce(oldval_num, newval_num) as numval,
processed_dttm
processed_dttm as &processed
from &ds1
where is_pk=1
order by key_hash, processed_dttm;
order by &keyhash, &processed;

/* grab pk values */
%local pk;
data _null_;
set &ds2;
by key_hash processed_dttm;
call symputx('pk',catx(' ',symget('pk'),tgtvar_nm),'l');
if last.processed_dttm then stop;
run;
select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;

%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
proc transpose data=&ds2(where=(tgtvar_type='C'))
out=&ds3(drop=_name_);
by KEY_HASH PROCESSED_DTTM;
by &keyhash &processed;
id TGTVAR_NM;
var charval;
run;

%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
proc transpose data=&ds2(where=(tgtvar_type='N'))
out=&ds4(drop=_name_);
by KEY_HASH PROCESSED_DTTM;
by &keyhash &processed;
id TGTVAR_NM;
var numval;
run;
Expand All @@ -13524,21 +13524,29 @@ run;
/* now merge to get all key values and de-dup */
%let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
data &ds5;
length key_hash $32 processed_dttm 8;
length &keyhash $32 &processed 8;
merge &ds3 &ds4;
by key_hash;
if not missing(key_hash);
by &keyhash &processed;
if not missing(&keyhash);
run;
proc sort data=&ds5 nodupkey;
by &pk;
run;

/* join to base table for preliminary stage DS */
proc sql;
create table &outds as select "No " as _____DELETE__THIS__RECORD_____,
b.*
create table &outds as select "No " as _____DELETE__THIS__RECORD_____
%do x=1 %to %sysfunc(countw(&pk,%str( )));
,a.%scan(&pk,&x,%str( ))
%end;
%local notpkcols;
%let notpkcols=%upcase(%mf_getvarlist(&libds));
%let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
%do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
,b.%scan(&notpkcols,&x,%str( ))
%end;
from &ds5 a
inner join &libds b
left join &libds b
on 1=1
%do x=1 %to %sysfunc(countw(&pk,%str( )));
and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
Expand Down Expand Up @@ -13587,13 +13595,23 @@ data _null_;
end;
else if move_type='D' then do;
if first.key_hash then do;
put "insert into &outds set _____DELETE__THIS__RECORD_____='No' " @@;
put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
end;
if IS_PK=0 then do;
put " ," tgtvar_nm '=' @@;
cnt=count(oldval_char,'"');
charval=quote(trim(substr(oldval_char,1,32765-cnt)));
if tgtvar_type='C' then put charval @@;
else put oldval_num @@;
end;
else do;
if first.is_pk then put " where 1=1 " @@;
put " and " tgtvar_nm '=' @@;
cnt=count(oldval_char,'"');
charval=quote(trim(substr(oldval_char,1,32765-cnt)));
if tgtvar_type='C' then put charval @@;
else put oldval_num @@;
end;
put " ," tgtvar_nm '=' @@;
cnt=count(oldval_char,'"');
charval=quote(trim(substr(oldval_char,1,32765-cnt)));
if tgtvar_type='C' then put charval @@;
else put oldval_num @@;
end;
if last.key_hash then put ';';
run;
Expand Down
64 changes: 41 additions & 23 deletions base/mp_stripdiffs.sas
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@
<h4> SAS Macros </h4>
@li mf_getuniquefileref.sas
@li mf_getuniquename.sas
@li mf_getvarlist.sas
@li mf_islibds.sas
@li mf_wordsinstr1butnotstr2.sas
@li mp_abort.sas
<h4> Related Macros </h4>
Expand Down Expand Up @@ -97,38 +99,36 @@ create table &ds1 (drop=libref dsn) as

/* extract key values only */
%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
%local keyhash processed;
%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
create table &ds2 as
select distinct key_hash,
select key_hash as &keyhash,
tgtvar_nm,
tgtvar_type,
coalescec(oldval_char,newval_char) as charval,
coalesce(oldval_num, newval_num) as numval,
processed_dttm
processed_dttm as &processed
from &ds1
where is_pk=1
order by key_hash, processed_dttm;
order by &keyhash, &processed;

/* grab pk values */
%local pk;
data _null_;
set &ds2;
by key_hash processed_dttm;
call symputx('pk',catx(' ',symget('pk'),tgtvar_nm),'l');
if last.processed_dttm then stop;
run;
select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;

%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
proc transpose data=&ds2(where=(tgtvar_type='C'))
out=&ds3(drop=_name_);
by KEY_HASH PROCESSED_DTTM;
by &keyhash &processed;
id TGTVAR_NM;
var charval;
run;

%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
proc transpose data=&ds2(where=(tgtvar_type='N'))
out=&ds4(drop=_name_);
by KEY_HASH PROCESSED_DTTM;
by &keyhash &processed;
id TGTVAR_NM;
var numval;
run;
Expand All @@ -139,21 +139,29 @@ run;
/* now merge to get all key values and de-dup */
%let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
data &ds5;
length key_hash $32 processed_dttm 8;
length &keyhash $32 &processed 8;
merge &ds3 &ds4;
by key_hash;
if not missing(key_hash);
by &keyhash &processed;
if not missing(&keyhash);
run;
proc sort data=&ds5 nodupkey;
by &pk;
run;

/* join to base table for preliminary stage DS */
proc sql;
create table &outds as select "No " as _____DELETE__THIS__RECORD_____,
b.*
create table &outds as select "No " as _____DELETE__THIS__RECORD_____
%do x=1 %to %sysfunc(countw(&pk,%str( )));
,a.%scan(&pk,&x,%str( ))
%end;
%local notpkcols;
%let notpkcols=%upcase(%mf_getvarlist(&libds));
%let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
%do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
,b.%scan(&notpkcols,&x,%str( ))
%end;
from &ds5 a
inner join &libds b
left join &libds b
on 1=1
%do x=1 %to %sysfunc(countw(&pk,%str( )));
and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
Expand Down Expand Up @@ -202,13 +210,23 @@ data _null_;
end;
else if move_type='D' then do;
if first.key_hash then do;
put "insert into &outds set _____DELETE__THIS__RECORD_____='No' " @@;
put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
end;
if IS_PK=0 then do;
put " ," tgtvar_nm '=' @@;
cnt=count(oldval_char,'"');
charval=quote(trim(substr(oldval_char,1,32765-cnt)));
if tgtvar_type='C' then put charval @@;
else put oldval_num @@;
end;
else do;
if first.is_pk then put " where 1=1 " @@;
put " and " tgtvar_nm '=' @@;
cnt=count(oldval_char,'"');
charval=quote(trim(substr(oldval_char,1,32765-cnt)));
if tgtvar_type='C' then put charval @@;
else put oldval_num @@;
end;
put " ," tgtvar_nm '=' @@;
cnt=count(oldval_char,'"');
charval=quote(trim(substr(oldval_char,1,32765-cnt)));
if tgtvar_type='C' then put charval @@;
else put oldval_num @@;
end;
if last.key_hash then put ';';
run;
Expand Down

0 comments on commit 22f0cb6

Please sign in to comment.