/********************************************************************************
* 
* FILE:    REDCap_split.sas
*
* VERSION: 0.0.0
*
* PURPOSE: Take a REDCap dataset with multiple events and make into several
*          tables with primary and foreign keys
*
* AUTHOR:  Paul W. Egeler, M.S., GStat
*
* DATE:    22JUN2017
*
*******************************************************************************
*
* INSTRUCTIONS:
* 
* 1. Run the SAS code provided by REDCap to load the records into your SAS session.
*
* 2. Download the data dictionary for your project.
*
* 3. Run the macro definitions REDCAP_READ_DATA_DICT and REDCAP_SPLIT
*
* 4. Run the macro call for REDCAP_READ_DATA_DICT to load in the data dictionry.
*    This is necessary to split the tables correctly.
* 
* 5. Run the macro call for REDCAP_SPLIT. You will have an output dataset for
*    your main table as well as for each repeating instrument.
*
********************************************************************************/

%MACRO REDCAP_READ_DATA_DICT(
    DATA_DICTIONARY  /* The file path for the data dictionary */
);
        DATA REDCAP_DATA_DICTIONARY;
    
            LENGTH VAR_NAME $ 200 FORM_NAME $ 200 SECTION_HEADER $ 200 FIELD_TYPE $ 200 X1-X14 $ 2250;
            INFILE "&DATA_DICTIONARY" FIRSTOBS = 2 DSD DLM = "," LRECL=32767;
            
            INPUT VAR_NAME $ FORM_NAME $ SECTION_HEADER $ FIELD_TYPE $ X1-X14 $;

            IF FIELD_TYPE EQ "descriptive" THEN DELETE;

            DROP SECTION_HEADER X1-X14;
                
        RUN;

%MEND REDCAP_READ_DATA_DICT;


%MACRO REDCAP_SPLIT(
    DATA_SET = REDCAP /* The name of the SAS dataset created by REDCap */,
    DATA_DICTIONARY = REDCAP_DATA_DICTIONARY  /* The name of the SAS dataset of the data dictionary */
);

    /* Remove formatting from repeat instrument field */
    DATA &DATA_SET.;
    SET &DATA_SET.;
      FORMAT REDCAP_REPEAT_INSTRUMENT;
    RUN;
    
    /* Find the subtable names and number of subtables */
    PROC SQL NOPRINT;

        SELECT DISTINCT
            REDCAP_REPEAT_INSTRUMENT,
            "'"!!trim(REDCAP_REPEAT_INSTRUMENT)!!"'" AS INSTRUMENT_QUOTED
        INTO 
            :INSTRUMENTS         SEPARATED BY ' ',
            :INSTRUMENT_LIST     SEPARATED BY ','
        FROM &DATA_SET AS A
        WHERE REDCAP_REPEAT_INSTRUMENT GT '';
        
        %LET N_SUBTABLES = &SQLOBS;

        %PUT INSTRUMENTS:     %LEFT(&INSTRUMENTS);
        %PUT INSTRUMENT LIST: %LEFT(&INSTRUMENT_LIST);
        %PUT N SUBTABLES:     %LEFT(&N_SUBTABLES);

    QUIT;
    

    %IF &N_SUBTABLES GT 0 %THEN %DO;
    
        /* Get information on the variables in the dataset */
        PROC CONTENTS 
          DATA = &DATA_SET. 
          OUT = REDCAP_VARNAMES(KEEP=NAME VARNUM)
          NOPRINT;
        RUN;

        /* Find the key that links the base table to child tables */
        DATA _NULL_;
        SET REDCAP_VARNAMES;
          IF VARNUM EQ 1 THEN DO;
            CALL SYMPUT("KEY", NAME);
            STOP;
          END;
        RUN;
        
        /* Make a list of fields and their associated forms based on data dictionary */
        DATA REDCAP_FIELDS(KEEP=VAR_NAME FORM_NAME);
        SET &DATA_DICTIONARY.;
          IF FIELD_TYPE EQ "checkbox" THEN DO;
            BASENAME = VAR_NAME;
            DO I = 1 TO N;
              SET REDCAP_VARNAMES POINT=I NOBS=N;
              IF PRXMATCH("/^"!!trim(BASENAME)!!"___.+$/", NAME) THEN DO;
                VAR_NAME = NAME;
                OUTPUT;
              END;
            END; 
          END;
          ELSE OUTPUT;
        RUN;

        /* Add instrument status fields to list of fields */
        PROC SQL;

        CREATE TABLE REDCAP_INSTRUMENT_STATUS_FIELDS AS
          SELECT DISTINCT TRIM(FORM_NAME)!!"_complete" AS VAR_NAME LENGTH=200, FORM_NAME
          FROM REDCAP_DATA_DICTIONARY;

        QUIT;

        PROC APPEND 
          BASE=REDCAP_FIELDS 
          DATA=REDCAP_INSTRUMENT_STATUS_FIELDS;
        RUN;
        
        /* Sort out the field names */
        PROC SQL NOPRINT;

            SELECT VAR_NAME
            INTO :VARS_BASE SEPARATED BY ' '
            FROM REDCAP_FIELDS AS A
            WHERE FORM_NAME NOT IN (&INSTRUMENT_LIST);

            %put Base vars: &VARS_BASE;
            
            %DO I = 1 %TO &N_SUBTABLES;

                %LET INSTRUMENT_I = %SCAN(&INSTRUMENTS,&I,%STR( ));
                
                SELECT VAR_NAME
                INTO :VARS_&INSTRUMENT_I. SEPARATED BY ' '
                FROM REDCAP_FIELDS AS A
                WHERE FORM_NAME EQ "&INSTRUMENT_I.";

                %put &INSTRUMENT_I. vars: &&VARS_&INSTRUMENT_I;
            
            %END;


        QUIT;

        /* Make new data sets based on field names above */
        DATA &DATA_SET._BASE (KEEP = &VARS_BASE);
            SET &DATA_SET;
            
            IF MISSING(REDCAP_REPEAT_INSTRUMENT);
        RUN;
            
        %DO I = 1 %TO &N_SUBTABLES;

            %LET INSTRUMENT_I = %SCAN(&INSTRUMENTS,&I,%STR( ));
        
            DATA &DATA_SET._&INSTRUMENT_I. (KEEP = &KEY redcap_repeat_instance &&VARS_&INSTRUMENT_I);
                SET &DATA_SET;
        
                IF REDCAP_REPEAT_INSTRUMENT EQ "&INSTRUMENT_I.";
            
            RUN;            
        
        %END;

    /* Clean up temporary datasets */
    PROC DATASETS MEMTYPE=DATA LIBRARY=WORK NOLIST;
    DELETE REDCAP_FIELDS REDCAP_VARNAMES REDCAP_INSTRUMENT_STATUS_FIELDS;
    RUN;

    %END;

    %ELSE %DO;

        %PUT THERE WERE NO REPEAT INSTRUMENTS IN THE DATASET %LEFT(&DATA_SET);
        %PUT NO ACTION WAS TAKEN;

    %END;


%MEND REDCAP_SPLIT;