From 4552dabe154aa731a873d25f6d0bc66de0d23174 Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Thu, 11 Nov 2021 14:48:43 -0500 Subject: [PATCH 01/20] Created iefieldkit_aux.do --- src/ado_files/iefieldkit_aux.do | 120 ++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 src/ado_files/iefieldkit_aux.do diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do new file mode 100644 index 00000000..e27b8c89 --- /dev/null +++ b/src/ado_files/iefieldkit_aux.do @@ -0,0 +1,120 @@ +/* + This do-file contains functions to perform the following tasks: +*/ + +/******************************************************************************* + GET FILE COMPONENTS + - Folder path + - File extension and file name +********************************************************************************/ +cap program drop ieaux_filename + program ieaux_filename, rclass + + syntax using/ + + * Standardize file path + local using = subinstr("`using'", "\", "/", .) + + * Separate the folder name from the file name + local r_lastslash = strlen(`"`using'"') - strpos(strreverse(`"`using'"'),"/") + if strpos(strreverse(`"`using'"'),"/") == 0 local r_lastslash -1 // Set to -1 if there is no slash + + * If a folder was not specified, get the folder path + local folder = substr(`"`using'"',1,`r_lastslash') + + + * Everything that comes after the folder path is the file name and format + local file = substr("`using'", `r_lastslash' + 2, .) + + * If a filename was specified, separate the file name from the file format + if "`file'" != "" { + + * Get index of separation between file name and file format + local r_lastsdot = strlen(`"`file'"') - strpos(strreverse( `"`file'"'),".") + + local fileext = substr(`"`file'"',`r_lastsdot'+1,.) // File format starts at the last period and ends at the end of the + if "`fileext'" == "" local fileext 1 + + local filename = substr("`file'", 1, `r_lastsdot') // File name starts at the beginning and ends at the last period + + } + + * If a format was not specified, the name is everything that follows the folder path + else { + local filename `file' + } + + * Check that a folder name was specified and through an error if it wasn't + if ("`file'" == "" | ("`filename'" == "")) { + noi di as error "{phang}`using' not a valid filename.{p_end}" + noi di "" + error 198 + } + + return local folder `folder' + return local file `file' + return local filename `filename' + return local fileext `fileext' + return local using `using' +end + + +/******************************************************************************* +TEST IF A FOLDER ALREADY EXISTS +- option "folderpath" is the path to the directory +- option "description" will be used in the error message to explain where this folder path was referenced + +********************************************************************************/ +cap program drop ieaux_folderpath + program ieaux_folderpath + + syntax, [description(string) folderpath(string)] + + * Test that the folder for the report file exists + mata : st_numscalar("r(dirExist)", direxists("`folderpath'")) + if `r(dirExist)' == 0 { + noi di as error `"{phang}The folder path [`folderpath'/] used `description' does not exist.{p_end}"' + error 601 + } + +end + + +/******************************************************************************* +TEST IF THE FILE EXTENSION IS THE CORRECT +- option "fileext" is a namelist of the correct extensions that the file may only have +- option "testfileext" is the current file extension + +********************************************************************************/ + +cap program drop ieaux_fileext + program ieaux_fileext, rclass + + syntax using/ , fileext(namelist) testfileext(string) + + * Check if the file extension is the correct + local ext "" + foreach value in `fileext' { + if (".`value'" == "`testfileext'") local errorfile 1 + local ext `".`value' `ext'"' + } + + local wcount = `: word count `fileext'' + if ("`errorfile'" != "1") & ("`testfileext'" != "1") { + if `wcount' > 1 local pluralms= "s" + noi di as error `"{phang}The file {bf:`using'} may only have the extension format`pluralms' [`ext']. The format [`testfileext'] is not allowed.{p_end}"' + error 198 + } + + + * If no file extension was used, then add the extension + if "`testfileext'" == "1" { + local ext = word("`fileext'",1) // If there are more than one extension, get first + local using "`using'.`ext'" + + } + return local using `using' +end + + + \ No newline at end of file From c752413cc9e87e69c190cc04ff81f3b1ef2cc070 Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Thu, 11 Nov 2021 14:53:21 -0500 Subject: [PATCH 02/20] iecodebook: use auxiliary functions to test files --- src/ado_files/iecodebook.ado | 45 +++++++++++------------------------- 1 file changed, 13 insertions(+), 32 deletions(-) diff --git a/src/ado_files/iecodebook.ado b/src/ado_files/iecodebook.ado index df96a51f..7bc6e954 100644 --- a/src/ado_files/iecodebook.ado +++ b/src/ado_files/iecodebook.ado @@ -28,39 +28,20 @@ cap program drop iecodebook // Select subcommand noi di " " gettoken subcommand anything : anything + + // Get the foldername, filename and the file extension + ieaux_filename using `using' - // Check folder exists - - // Start by standardize all slashes to forward slashes, and get the position of the last slash - local using = subinstr("`using'","\","/",.) - local r_lastslash = strlen(`"`using'"') - strpos(strreverse(`"`using'"'),"/") - if strpos(strreverse(`"`using'"'),"/") == 0 local r_lastslash -1 // Set to -1 if there is no slash - - // Get the full folder path and the file name - local r_folder = substr(`"`using'"',1,`r_lastslash') - local r_file = substr(`"`using'"',`r_lastslash'+2,.) - - // Test that the folder for the report file exists - mata : st_numscalar("r(dirExist)", direxists("`r_folder'")) - if `r(dirExist)' == 0 { - noi di as error `"{phang}The folder [`r_folder'/] does not exist.{p_end}"' - error 601 - } - - // Find the position of the last dot in the file name and get the file format extension - local r_lastsdot = strlen(`"`r_file'"') - strpos(strreverse(`"`r_file'"'),".") - local r_fileextension = substr(`"`r_file'"',`r_lastsdot'+1,.) - - // If no fileextension was used, then add .xslx to "`using'" - if "`r_fileextension'" == "" { - local using "`using'.xlsx" - } - // Throw an error if user input uses any extension other than the allowed - else if !inlist("`r_fileextension'",".xlsx",".xls") & !regexm(`"`options'"',"tempfile") { - di as error "The codebook may only have the file extension [.xslx] or [.xls]. The format [`r_fileextension'] is not allowed." - error 601 - } - + //Standarize path + local using "`r(using)'" + + // Test that the folder exists + ieaux_folderpath, folderpath("`r(folder)'") + + + // Test the form file: xls or xlsx + ieaux_fileext using `using', fileext(xlsx xls) testfileext("`r(fileext)'") + local using "`r(using)'" // Throw error on [template] if codebook cannot be created if inlist("`subcommand'","template","export") & !regexm(`"`options'"',"replace") & !regexm(`"`options'"',"verify") { From c69302b4b36ba3a52d301f93c6b76e248f00497e Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Thu, 11 Nov 2021 14:53:43 -0500 Subject: [PATCH 03/20] ieduplicates: use auxiliary functions to test files --- src/ado_files/ieduplicates.ado | 89 ++++++++-------------------------- 1 file changed, 19 insertions(+), 70 deletions(-) diff --git a/src/ado_files/ieduplicates.ado b/src/ado_files/ieduplicates.ado index 533a3946..3eec0fd7 100644 --- a/src/ado_files/ieduplicates.ado +++ b/src/ado_files/ieduplicates.ado @@ -919,68 +919,28 @@ cap program drop testpath Prepare file name if option using was specified *******************************************************************************/ - * Parse using option to get (1) the folder path (2) the name of the report and - * (3) the format selected else if "`using'" != "" { - - * Replace any backslashes with forward slashes so it's compatible with - * different OS and so we know what to look for as separators when parsing - local using = subinstr("`using'", "\", "/", .) - - * Separate the folder name from the file name - strlast, expression("`using'") character("/") - - * If a folder was specified, get the folder path - if `r(lastpos)' > 0 { - local folder = substr("`using'", 1, `r(lastpos)') - } - else { + *get the folder path, the name of the report and, the format selected + ieaux_filename using `using' + local folder `r(folder)'/ + local name `r(filename)' + local ext `r(fileext)' + + * Standarize path + local using "`r(using)'" + + * Check if the folder exist + if missing("`folder'") { noi di as error "{phang}You have not specified a folder path to the duplicates report. An absolute folder path is required.{p_end}" noi di as error `"{phang}This command will not work if you are trying to use {inp:cd} to set the directory and open or save files. To know more about why this practice is not allowed, {browse "https://dimewiki.worldbank.org/wiki/Stata_Coding_Practices#File_paths":see this article in the DIME Wiki}.{p_end}"' noi di as error "" error 198 - exit - } - - * Everything that comas after the folder path is the file name and format - local file = substr("`using'", `r(lastpos)' + 1, .) - - * If a filename was specified, separate the file name from the file format - if "`file'" != "" { - - * Get index of separation between file name and file format - strlast, expression("`file'") character(".") - - * If a format was specified, separate name and format - if `r(lastpos)' > 0 { - local ext = substr("`file'", `r(lastpos)', .) // File format starts at the last period and ends at the end of the string - local name = substr("`file'", 1, `r(lastpos)' - 1) // File name starts at the beginning and ends at the last period - } - * If a format was not specified, the name is everything that follows the - * folder path - else { - local name `file' - } - } - * Check that a folder name was specified and through an error if it wasn't - if ("`file'" == "" | ( "`file'" != "" & "`name'" == "")) { - noi di as error "{phang}`using' not a valid filename.{p_end}" - noi di "" - error 198 - exit - } - - * The default format is xlsx. Other possible formats are xls - if "`ext'" == "" { - local ext .xlsx - } - else if !inlist("`ext'", ".xls", ".xlsx") { - noi di "" - noi di as error `"{phang}`ext' is not currently supported as a format for the duplicates report. Supported formats are: xls, xslx. If you have a suggestion of a different format to support, please e-mail dimeanalytics@worldbank.org or {browse "https://github.com/worldbank/iefieldkit/issues":create an issue on the iefieldkit GitHub repository.}{p_end}"' - noi di "" - error 198 //TODO: check error code - exit } + + * Check if the file extension is the correct. + ieaux_fileext using `using', fileext(xlsx xls) testfileext(`ext') + local using "`r(using)'" + } * Create file name if using was not specified @@ -991,24 +951,13 @@ cap program drop testpath if "`suffix'" != "" local name `name'_`suffix' local ext .xlsx } - - * Test that the folder indicated existes - mata : st_numscalar("r(dirExist)", direxists("`folder'")) - - ** If the folder does not exist, throw an error - if `r(dirExist)' == 0 { - - *Variable exist, output error - noi di as error "{phang}The folder specified does not exist :`folder'.{p_end}" - noi di "" - error 198 // TODO: check error code - exit - } + + * Test that the folder exists + ieaux_folderpath, folderpath("`folder'") return local name `name' return local ext `ext' return local folder `folder' - end /******************************************************************************* From 8234e8def6512e7774c17e7732663aa552836d4b Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Thu, 11 Nov 2021 14:54:19 -0500 Subject: [PATCH 04/20] ietestform: use auxiliary functions to test files --- src/ado_files/ietestform.ado | 73 ++++++++++++------------------------ 1 file changed, 23 insertions(+), 50 deletions(-) diff --git a/src/ado_files/ietestform.ado b/src/ado_files/ietestform.ado index d1af5a42..76862e7a 100644 --- a/src/ado_files/ietestform.ado +++ b/src/ado_files/ietestform.ado @@ -3,8 +3,7 @@ capture program drop ietestform program ietestform , rclass -qui { - +qui{ version 13 preserve @@ -30,15 +29,20 @@ qui { } local surveyform `"`using'`surveyform'"' + *Get the foldername, filename and the file extension for the survey file input + ieaux_filename using `surveyform' - * Test for form file is xls or xlsx - local surveyformtype = substr(`"`surveyform'"',strlen(`"`surveyform'"')-strpos(strreverse(`"`surveyform'"'),".")+1,.) - if !(`"`surveyformtype'"' == ".xls" | `"`surveyformtype'"' == ".xlsx") { - noi di as error `"{phang}The survey form file [`surveyform'] must have file extension .xls or .xlsx specified in the option.{p_end}"' - error 601 - } + * Standarize path + local surveyform "`r(using)'" + + *Test that the folder exists + ieaux_folderpath, folderpath("`r(folder)'") description("in surveyform ") + + * Test if the form file is xls or xlsx + ieaux_fileext using `surveyform', fileext(xlsx xls) testfileext("`r(fileext)'") + local surveyform "`r(using)'" - *Test that the form file exists + *Test if the form file exists cap confirm file "`surveyform'" if _rc { @@ -51,49 +55,18 @@ qui { *********/ ********* - *Get the folder for the report file - - **Start by finding the position of the last forward slash. If no forward - * slash exist, it is zero, then replace to to string len so it is never - * the min() below. - local r_f_slash = strpos(strreverse(`"`reportsave'"'),"\") - if `r_f_slash' == 0 local r_f_slash = strlen(`"`reportsave'"') - - **Start by finding the position of the last backward slash. If no backward - * slash exist, it is zero, then replace to to string len so it is never - * the min() below. - local r_b_slash = strpos(strreverse(`"`reportsave'"'),"/") - if `r_b_slash' == 0 local r_b_slash = strlen(`"`reportsave'"') - - *Get the last slash in the report file path regardless of back or forward - local r_lastslash = strlen(`"`reportsave'"')-min(`r_f_slash',`r_b_slash') - - *Get the folder - local r_folder = substr(`"`reportsave'"',1,`r_lastslash') + *Get the foldername, filename and the file extension for the report file + ieaux_filename using `using' + + * Standarize path + local using "`r(using)'" *Test that the folder for the report file exists - mata : st_numscalar("r(dirExist)", direxists("`r_folder'")) - if `r(dirExist)' == 0 { - noi di as error `"{phang}The folder used in [`reportsave'] does not exist.{p_end}"' - error 601 - } - - *Get the filename and the file extension type from the report file - local r_filename = substr(`"`reportsave'"',`r_lastslash'+1, .) - local r_filenametype = substr(`"`r_filename'"',strlen(`"`r_filename'"')-strpos(strreverse(`"`r_filename'"'),".")+1,.) - *Test what the file extension type is - if (`"`r_filenametype'"' == "") { - *No file type specified, add .csv - local reportsave `"`reportsave'.csv"' - } - else if (`"`r_filenametype'"' != ".csv") { - *Incorrect file type added. Throw error - noi di as error `"{phang}The report file [`reportsave'] may only have the file extension .csv.{p_end}"' - error 601 - } - else { - * All is correct, do nothing - } + ieaux_folderpath, folderpath("`r(folder)'") description("in [`reportsave'] ") + + *Test if the file extension type from the report file is csv + ieaux_fileext `using', fileext(csv) testfileext(`r_fileext') + local using "`r(using)'" *Tempfile that will be used to write the report tempfile report_tempfile From 553c0ae49e18215c8a3544eb8c437b669a103966 Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Sun, 28 Nov 2021 19:01:39 -0500 Subject: [PATCH 05/20] iefieldkit_aux: Remove some conditions from ieaux_filename functions --- src/ado_files/iefieldkit_aux.do | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do index e27b8c89..5b1e4b7d 100644 --- a/src/ado_files/iefieldkit_aux.do +++ b/src/ado_files/iefieldkit_aux.do @@ -22,7 +22,6 @@ cap program drop ieaux_filename * If a folder was not specified, get the folder path local folder = substr(`"`using'"',1,`r_lastslash') - * Everything that comes after the folder path is the file name and format local file = substr("`using'", `r_lastslash' + 2, .) @@ -39,18 +38,6 @@ cap program drop ieaux_filename } - * If a format was not specified, the name is everything that follows the folder path - else { - local filename `file' - } - - * Check that a folder name was specified and through an error if it wasn't - if ("`file'" == "" | ("`filename'" == "")) { - noi di as error "{phang}`using' not a valid filename.{p_end}" - noi di "" - error 198 - } - return local folder `folder' return local file `file' return local filename `filename' From 9cac6b261f68f7acf45ed9cd4bd5f90257086265 Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Mon, 29 Nov 2021 06:19:05 -0500 Subject: [PATCH 06/20] iefieldkit_aux: Change ieaux_folderpath function syntaxis --- src/ado_files/iefieldkit_aux.do | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do index 5b1e4b7d..b669b347 100644 --- a/src/ado_files/iefieldkit_aux.do +++ b/src/ado_files/iefieldkit_aux.do @@ -37,7 +37,7 @@ cap program drop ieaux_filename local filename = substr("`file'", 1, `r_lastsdot') // File name starts at the beginning and ends at the last period } - + return local folder `folder' return local file `file' return local filename `filename' @@ -55,12 +55,12 @@ TEST IF A FOLDER ALREADY EXISTS cap program drop ieaux_folderpath program ieaux_folderpath - syntax, [description(string) folderpath(string)] + syntax using/, [description(string)] * Test that the folder for the report file exists - mata : st_numscalar("r(dirExist)", direxists("`folderpath'")) + mata : st_numscalar("r(dirExist)", direxists("`using'")) if `r(dirExist)' == 0 { - noi di as error `"{phang}The folder path [`folderpath'/] used `description' does not exist.{p_end}"' + noi di as error `"{phang}The folder path [`using'/] used `description' does not exist.{p_end}"' error 601 } From ce9021e5a0d1dbc30455b70113eaabc57adf3e91 Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Mon, 29 Nov 2021 07:38:36 -0500 Subject: [PATCH 07/20] iefieldkit: Check that a filename was specified --- src/ado_files/iefieldkit_aux.do | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do index b669b347..2c633052 100644 --- a/src/ado_files/iefieldkit_aux.do +++ b/src/ado_files/iefieldkit_aux.do @@ -37,6 +37,13 @@ cap program drop ieaux_filename local filename = substr("`file'", 1, `r_lastsdot') // File name starts at the beginning and ends at the last period } + + * Check that a filename was specified and through an error if it wasn't + if "`file'" == "" { + noi di as error "{phang}`using' not a valid filename.{p_end}" + noi di "" + error 198 + } return local folder `folder' return local file `file' From 506a71ee9bbcf09bec5e6253a5ada6cc0ca8085c Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Tue, 30 Nov 2021 09:11:43 -0500 Subject: [PATCH 08/20] iefieldkit_aux: Add ieaux_filename command in the others commands --- src/ado_files/iefieldkit_aux.do | 38 +++++++++++++++++---------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do index 2c633052..2c8c236d 100644 --- a/src/ado_files/iefieldkit_aux.do +++ b/src/ado_files/iefieldkit_aux.do @@ -23,7 +23,7 @@ cap program drop ieaux_filename local folder = substr(`"`using'"',1,`r_lastslash') * Everything that comes after the folder path is the file name and format - local file = substr("`using'", `r_lastslash' + 2, .) + local file = substr("`using'", `r_lastslash' + 2, .) * If a filename was specified, separate the file name from the file format if "`file'" != "" { @@ -63,47 +63,49 @@ cap program drop ieaux_folderpath program ieaux_folderpath syntax using/, [description(string)] + ieaux_filename using `using' * Test that the folder for the report file exists - mata : st_numscalar("r(dirExist)", direxists("`using'")) - if `r(dirExist)' == 0 { - noi di as error `"{phang}The folder path [`using'/] used `description' does not exist.{p_end}"' - error 601 - } + if !missing("`r(folder)'"){ + mata : st_numscalar("r(dirExist)", direxists("`r(folder)'")) + if `r(dirExist)' == 0 { + noi di as error `"{phang}The folder path [`r(folder)'/] used `description' does not exist.{p_end}"' + error 601 + } + } end /******************************************************************************* TEST IF THE FILE EXTENSION IS THE CORRECT -- option "fileext" is a namelist of the correct extensions that the file may only have -- option "testfileext" is the current file extension +- option "testfileext" is a namelist of the correct extensions that the file may only have ********************************************************************************/ cap program drop ieaux_fileext program ieaux_fileext, rclass - syntax using/ , fileext(namelist) testfileext(string) + syntax using/, testfileext(string) + ieaux_filename using `using' * Check if the file extension is the correct local ext "" - foreach value in `fileext' { - if (".`value'" == "`testfileext'") local errorfile 1 - local ext `".`value' `ext'"' + foreach value in `testfileext' { + if (".`value'" == "`r(fileext)'") local errorfile 1 + local ext `".`value' `ext'"' } - local wcount = `: word count `fileext'' - if ("`errorfile'" != "1") & ("`testfileext'" != "1") { + local wcount = `: word count `testfileext'' + if ("`errorfile'" != "1") & ("`r(fileext)'" != "1") { if `wcount' > 1 local pluralms= "s" - noi di as error `"{phang}The file {bf:`using'} may only have the extension format`pluralms' [`ext']. The format [`testfileext'] is not allowed.{p_end}"' + noi di as error `"{phang}The file {bf:`using'} may only have the extension format`pluralms' [`ext']. The format [`r(fileext)'] is not allowed.{p_end}"' error 198 } - * If no file extension was used, then add the extension - if "`testfileext'" == "1" { - local ext = word("`fileext'",1) // If there are more than one extension, get first + if "`r(fileext)'" == "1" { + local ext = word("`testfileext'",1) // If there are more than one extension, get first local using "`using'.`ext'" } From 2d80a4b825c2d18d99c020aa58e356fecb205198 Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Wed, 1 Dec 2021 15:11:30 -0500 Subject: [PATCH 09/20] iefieldkit_aux: Drop filename condition in ieaux_filename --- src/ado_files/iefieldkit_aux.do | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do index 2c8c236d..80850aa7 100644 --- a/src/ado_files/iefieldkit_aux.do +++ b/src/ado_files/iefieldkit_aux.do @@ -38,13 +38,6 @@ cap program drop ieaux_filename } - * Check that a filename was specified and through an error if it wasn't - if "`file'" == "" { - noi di as error "{phang}`using' not a valid filename.{p_end}" - noi di "" - error 198 - } - return local folder `folder' return local file `file' return local filename `filename' From 7279d5ceb971c34f3f5b73aae4ccd6de952338bf Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Wed, 1 Dec 2021 15:26:18 -0500 Subject: [PATCH 10/20] iefieldkit_aux: Test if there is a filename --- src/ado_files/iefieldkit_aux.do | 39 +++++++++++++++++---------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do index 80850aa7..970eff6b 100644 --- a/src/ado_files/iefieldkit_aux.do +++ b/src/ado_files/iefieldkit_aux.do @@ -82,26 +82,27 @@ cap program drop ieaux_fileext syntax using/, testfileext(string) ieaux_filename using `using' - * Check if the file extension is the correct - local ext "" - foreach value in `testfileext' { - if (".`value'" == "`r(fileext)'") local errorfile 1 - local ext `".`value' `ext'"' - } - - local wcount = `: word count `testfileext'' - if ("`errorfile'" != "1") & ("`r(fileext)'" != "1") { - if `wcount' > 1 local pluralms= "s" - noi di as error `"{phang}The file {bf:`using'} may only have the extension format`pluralms' [`ext']. The format [`r(fileext)'] is not allowed.{p_end}"' - error 198 - } - - * If no file extension was used, then add the extension - if "`r(fileext)'" == "1" { - local ext = word("`testfileext'",1) // If there are more than one extension, get first - local using "`using'.`ext'" + if !missing("`r(file)'") { + * Check if the file extension is the correct + local ext "" + foreach value in `testfileext' { + if (".`value'" == "`r(fileext)'") local errorfile 1 + local ext `".`value' `ext'"' + } - } + local wcount = `: word count `testfileext'' + if ("`errorfile'" != "1") & ("`r(fileext)'" != "1") { + if `wcount' > 1 local pluralms= "s" + noi di as error `"{phang}The file {bf:`using'} may only have the extension format`pluralms' [`ext']. The format [`r(fileext)'] is not allowed.{p_end}"' + error 198 + } + + * If no file extension was used, then add the extension + if "`r(fileext)'" == "1" { + local ext = word("`testfileext'",1) // If there are more than one extension, get first + local using "`using'.`ext'" + } + } return local using `using' end From a5fe65a417baf16a740b5256b1358ed85e713f10 Mon Sep 17 00:00:00 2001 From: Denisse Orozco <48063061+DenisseO@users.noreply.github.com> Date: Wed, 1 Dec 2021 16:10:06 -0500 Subject: [PATCH 11/20] iefieldkit_aux: minor change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kristoffer Bjärkefur --- src/ado_files/iefieldkit_aux.do | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do index 970eff6b..b678f2f1 100644 --- a/src/ado_files/iefieldkit_aux.do +++ b/src/ado_files/iefieldkit_aux.do @@ -19,7 +19,7 @@ cap program drop ieaux_filename local r_lastslash = strlen(`"`using'"') - strpos(strreverse(`"`using'"'),"/") if strpos(strreverse(`"`using'"'),"/") == 0 local r_lastslash -1 // Set to -1 if there is no slash - * If a folder was not specified, get the folder path + * If a folder was specified, get the folder path local folder = substr(`"`using'"',1,`r_lastslash') * Everything that comes after the folder path is the file name and format From ad01494aa477b406797bb1859afe6e0d53dcf5b5 Mon Sep 17 00:00:00 2001 From: Denisse Orozco <48063061+DenisseO@users.noreply.github.com> Date: Wed, 1 Dec 2021 16:11:23 -0500 Subject: [PATCH 12/20] iefieldkit_aux: Add minor description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kristoffer Bjärkefur --- src/ado_files/iefieldkit_aux.do | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do index b678f2f1..882d77e3 100644 --- a/src/ado_files/iefieldkit_aux.do +++ b/src/ado_files/iefieldkit_aux.do @@ -12,7 +12,7 @@ cap program drop ieaux_filename syntax using/ - * Standardize file path + * Standardize file path so Mac and Linux systems handle this path correctly local using = subinstr("`using'", "\", "/", .) * Separate the folder name from the file name From 68605b2fb7f13810cc3b45ea9e024efdfeb0614c Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Tue, 7 Dec 2021 09:27:50 -0500 Subject: [PATCH 13/20] iefieldkit_aux: Update ieaux_fileext --- src/ado_files/iefieldkit_aux.do | 61 ++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do index 882d77e3..33dadfb3 100644 --- a/src/ado_files/iefieldkit_aux.do +++ b/src/ado_files/iefieldkit_aux.do @@ -32,7 +32,6 @@ cap program drop ieaux_filename local r_lastsdot = strlen(`"`file'"') - strpos(strreverse( `"`file'"'),".") local fileext = substr(`"`file'"',`r_lastsdot'+1,.) // File format starts at the last period and ends at the end of the - if "`fileext'" == "" local fileext 1 local filename = substr("`file'", 1, `r_lastsdot') // File name starts at the beginning and ends at the last period @@ -57,7 +56,7 @@ cap program drop ieaux_folderpath syntax using/, [description(string)] ieaux_filename using `using' - + * Test that the folder for the report file exists if !missing("`r(folder)'"){ mata : st_numscalar("r(dirExist)", direxists("`r(folder)'")) @@ -79,31 +78,53 @@ TEST IF THE FILE EXTENSION IS THE CORRECT cap program drop ieaux_fileext program ieaux_fileext, rclass - syntax using/, testfileext(string) + syntax using/, allowed_exts(string) [default_ext(string)] + + *Parse the input ieaux_filename using `using' + local this_filename "`r(filename)'" + local this_ext "`r(fileext)'" - if !missing("`r(file)'") { - * Check if the file extension is the correct - local ext "" - foreach value in `testfileext' { - if (".`value'" == "`r(fileext)'") local errorfile 1 - local ext `".`value' `ext'"' - } + *Test if using has no file + if missing("`this_filename'") { + noi di as error `"{phang}The path {bf:`using'} does not have a file name for which extension can be tested.{p_end}"' + error 198 + } + *Test is using has no file extension + else if missing("`this_ext'") { - local wcount = `: word count `testfileext'' - if ("`errorfile'" != "1") & ("`r(fileext)'" != "1") { - if `wcount' > 1 local pluralms= "s" - noi di as error `"{phang}The file {bf:`using'} may only have the extension format`pluralms' [`ext']. The format [`r(fileext)'] is not allowed.{p_end}"' - error 198 + *Test if no deafult ext was provided + if missing("`default_ext'") { + noi di as error `"{phang}The file in {bf:`using'} does not have a file extension and no default was provided.{p_end}"' + error 198 + } + + *Apply the deafult extension + else { + + local return_file "`using'`default_ext'" + } + } + + * Using has both file and extension + else { - * If no file extension was used, then add the extension - if "`r(fileext)'" == "1" { - local ext = word("`testfileext'",1) // If there are more than one extension, get first - local using "`using'.`ext'" + * Test if extension is among the allowed extensions + if `: list this_ext in allowed_exts' == 1 { + *Extension is allowed, return file name as is + local return_file "`using'" + } + + * File extension used is not allowed + else { + noi di as error `"{phang}The file extension [`this_ext'] in file {bf:`using'} is not allowed. Allowed extensions: [`allowed_exts'].{p_end}"' + error 198 } } - return local using `using' + + *Return checked filename + return local filename `return_file' end From 39766c3ff112f66ddc978a12b586c678140547f6 Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Tue, 7 Dec 2021 09:33:34 -0500 Subject: [PATCH 14/20] ieduplicates: Update iefieldkit_aux commands --- src/ado_files/ieduplicates.ado | 35 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/ado_files/ieduplicates.ado b/src/ado_files/ieduplicates.ado index 3eec0fd7..03f4e569 100644 --- a/src/ado_files/ieduplicates.ado +++ b/src/ado_files/ieduplicates.ado @@ -4,7 +4,7 @@ program ieduplicates , rclass - qui { + qui { syntax varname [using/] , UNIQUEvars(varlist) /// [force /// @@ -30,7 +30,6 @@ * message to this that is more informative. preserve - /*********************************************************************** ************************************************************************ Section 1 - Set up locals needed in data @@ -854,7 +853,7 @@ * the orignal data set in case of error. use `dataToReturn', clear - } + } end @@ -920,16 +919,10 @@ cap program drop testpath *******************************************************************************/ else if "`using'" != "" { - *get the folder path, the name of the report and, the format selected - ieaux_filename using `using' - local folder `r(folder)'/ - local name `r(filename)' - local ext `r(fileext)' - - * Standarize path - local using "`r(using)'" * Check if the folder exist + ieaux_filename using `using' + local folder `r(folder)' if missing("`folder'") { noi di as error "{phang}You have not specified a folder path to the duplicates report. An absolute folder path is required.{p_end}" noi di as error `"{phang}This command will not work if you are trying to use {inp:cd} to set the directory and open or save files. To know more about why this practice is not allowed, {browse "https://dimewiki.worldbank.org/wiki/Stata_Coding_Practices#File_paths":see this article in the DIME Wiki}.{p_end}"' @@ -938,8 +931,17 @@ cap program drop testpath } * Check if the file extension is the correct. - ieaux_fileext using `using', fileext(xlsx xls) testfileext(`ext') - local using "`r(using)'" + ieaux_fileext using `using', allowed_exts(.xlsx .xls) default_ext(.xlsx) + local using "`r(filename)'" + + * Test that the folder exists + ieaux_folderpath using `using' + + * Get the folder path, the name of the report and, the format selected + ieaux_filename using `using' + local folder `r(folder)'/ + local name `r(filename)' + local ext `r(fileext)' } @@ -952,12 +954,11 @@ cap program drop testpath local ext .xlsx } - * Test that the folder exists - ieaux_folderpath, folderpath("`folder'") - + return local name `name' return local ext `ext' - return local folder `folder' + return local folder `folder' + end /******************************************************************************* From 12c57c8c3434c215e8fcd0866ed99ad727cc8bfa Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Tue, 7 Dec 2021 10:49:48 -0500 Subject: [PATCH 15/20] iefieldkit_aux: Change to ieutil_ commands names --- src/ado_files/iefieldkit_aux.do | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do index 33dadfb3..0c3ffa1d 100644 --- a/src/ado_files/iefieldkit_aux.do +++ b/src/ado_files/iefieldkit_aux.do @@ -7,8 +7,8 @@ - Folder path - File extension and file name ********************************************************************************/ -cap program drop ieaux_filename - program ieaux_filename, rclass +cap program drop ieutil_parse_filepath + program ieutil_parse_filepath, rclass syntax using/ @@ -51,11 +51,11 @@ TEST IF A FOLDER ALREADY EXISTS - option "description" will be used in the error message to explain where this folder path was referenced ********************************************************************************/ -cap program drop ieaux_folderpath - program ieaux_folderpath +cap program drop ieutil__folderpath + program ieutil__folderpath syntax using/, [description(string)] - ieaux_filename using `using' + ieutil_parse_filepath using `using' * Test that the folder for the report file exists if !missing("`r(folder)'"){ @@ -75,13 +75,13 @@ TEST IF THE FILE EXTENSION IS THE CORRECT ********************************************************************************/ -cap program drop ieaux_fileext - program ieaux_fileext, rclass +cap program drop ieutil_fileext + program ieutil_fileext, rclass syntax using/, allowed_exts(string) [default_ext(string)] *Parse the input - ieaux_filename using `using' + ieutil_parse_filepath using `using' local this_filename "`r(filename)'" local this_ext "`r(fileext)'" From 3dd13a6a4a63148f860dd5d490f3d73977e6d89a Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Tue, 7 Dec 2021 10:59:42 -0500 Subject: [PATCH 16/20] iefieldkit_aux: Minor change --- src/ado_files/iefieldkit_aux.do | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do index 0c3ffa1d..9d916214 100644 --- a/src/ado_files/iefieldkit_aux.do +++ b/src/ado_files/iefieldkit_aux.do @@ -51,8 +51,8 @@ TEST IF A FOLDER ALREADY EXISTS - option "description" will be used in the error message to explain where this folder path was referenced ********************************************************************************/ -cap program drop ieutil__folderpath - program ieutil__folderpath +cap program drop ieutil_folderpath + program ieutil_folderpath syntax using/, [description(string)] ieutil_parse_filepath using `using' From c19f47c11d6290865f41e0c2b7b579f610e0dd50 Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Thu, 9 Dec 2021 05:47:08 -0500 Subject: [PATCH 17/20] iefieldkit_aux: Minor change --- src/ado_files/iefieldkit_aux.do | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/ado_files/iefieldkit_aux.do b/src/ado_files/iefieldkit_aux.do index 9d916214..97a66cec 100644 --- a/src/ado_files/iefieldkit_aux.do +++ b/src/ado_files/iefieldkit_aux.do @@ -37,11 +37,11 @@ cap program drop ieutil_parse_filepath } - return local folder `folder' - return local file `file' - return local filename `filename' - return local fileext `fileext' - return local using `using' + return local folderpath `folder' + return local file `file' + return local filename `filename' + return local fileext `fileext' + return local filepath `using' end @@ -58,10 +58,10 @@ cap program drop ieutil_folderpath ieutil_parse_filepath using `using' * Test that the folder for the report file exists - if !missing("`r(folder)'"){ - mata : st_numscalar("r(dirExist)", direxists("`r(folder)'")) + if !missing("`r(folderpath)'"){ + mata : st_numscalar("r(dirExist)", direxists("`r(folderpath)'")) if `r(dirExist)' == 0 { - noi di as error `"{phang}The folder path [`r(folder)'/] used `description' does not exist.{p_end}"' + noi di as error `"{phang}The folder path [`r(folderpath)'/] used `description' does not exist.{p_end}"' error 601 } } @@ -124,7 +124,7 @@ cap program drop ieutil_fileext } *Return checked filename - return local filename `return_file' + return local file_path `return_file' end From 1c04e1f02bdbba573e986690543ad9f2b1c4bbcc Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Fri, 10 Dec 2021 06:57:50 -0500 Subject: [PATCH 18/20] iecodebook: update ieutil_ commands --- src/ado_files/iecodebook.ado | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/ado_files/iecodebook.ado b/src/ado_files/iecodebook.ado index 7bc6e954..e08e73b7 100644 --- a/src/ado_files/iecodebook.ado +++ b/src/ado_files/iecodebook.ado @@ -4,6 +4,8 @@ cap program drop iecodebook program def iecodebook + + qui do "${GitHub}\iefieldkit\src\ado_files/iefieldkit_aux.do" version 13 // Requires 13.0 due to use of long macros @@ -29,29 +31,23 @@ cap program drop iecodebook noi di " " gettoken subcommand anything : anything - // Get the foldername, filename and the file extension - ieaux_filename using `using' - - //Standarize path - local using "`r(using)'" - // Test that the folder exists - ieaux_folderpath, folderpath("`r(folder)'") + ieutil_folderpath using `using', description("in the file") // Test the form file: xls or xlsx - ieaux_fileext using `using', fileext(xlsx xls) testfileext("`r(fileext)'") - local using "`r(using)'" + ieutil_fileext using `using', allowed_exts(.xlsx .xls) default_ext(.xlsx) + local using "`r(file_path)'" + // Throw error on [template] if codebook cannot be created if inlist("`subcommand'","template","export") & !regexm(`"`options'"',"replace") & !regexm(`"`options'"',"verify") { - + cap confirm file "`using'" if (_rc == 0) & (!strpos(`"`options'"',"replace")) { di as err "That codebook already exists. {bf:iecodebook} will only overwrite it if you specify the [replace] option." error 602 } - cap confirm new file "`using'" if (_rc != 0) & (!strpos(`"`options'"',"replace")) { di as error "{bf:iecodebook} could not create file `using'. Check that the file path is correctly specified." From 54777227e0208f1718648e4c36f38cacf2a568a9 Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Fri, 10 Dec 2021 07:02:29 -0500 Subject: [PATCH 19/20] ieduplicates: update ieutil_ commands --- src/ado_files/ieduplicates.ado | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/ado_files/ieduplicates.ado b/src/ado_files/ieduplicates.ado index 03f4e569..aba14a8d 100644 --- a/src/ado_files/ieduplicates.ado +++ b/src/ado_files/ieduplicates.ado @@ -17,7 +17,9 @@ duplistid(string) datelisted(string) datefixed(string) correct(string) drop(string) newid(string) initials(string) notes(string) listofdiffs(string)] version 11.0 - + + qui do "${GitHub}\iefieldkit\src\ado_files/iefieldkit_aux.do" + *Add version of Stata fix //Make sure that keepvars are still saved if saved if the duplicates file * is generated on a subset of the data. For example, duplicates from @@ -853,7 +855,7 @@ * the orignal data set in case of error. use `dataToReturn', clear - } + } end @@ -921,8 +923,8 @@ cap program drop testpath else if "`using'" != "" { * Check if the folder exist - ieaux_filename using `using' - local folder `r(folder)' + ieutil_parse_filepath using `using' + local folder `r(folderpath)' if missing("`folder'") { noi di as error "{phang}You have not specified a folder path to the duplicates report. An absolute folder path is required.{p_end}" noi di as error `"{phang}This command will not work if you are trying to use {inp:cd} to set the directory and open or save files. To know more about why this practice is not allowed, {browse "https://dimewiki.worldbank.org/wiki/Stata_Coding_Practices#File_paths":see this article in the DIME Wiki}.{p_end}"' @@ -931,17 +933,17 @@ cap program drop testpath } * Check if the file extension is the correct. - ieaux_fileext using `using', allowed_exts(.xlsx .xls) default_ext(.xlsx) - local using "`r(filename)'" + ieutil_fileext using `using', allowed_exts(.xlsx .xls) default_ext(.xlsx) + local using "`r(file_path)'" * Test that the folder exists - ieaux_folderpath using `using' + ieutil_folderpath using `using' * Get the folder path, the name of the report and, the format selected - ieaux_filename using `using' - local folder `r(folder)'/ - local name `r(filename)' - local ext `r(fileext)' + ieutil_parse_filepath using `using' + local folder `r(folderpath)'/ + local name `r(filename)' + local ext `r(fileext)' } @@ -954,7 +956,6 @@ cap program drop testpath local ext .xlsx } - return local name `name' return local ext `ext' return local folder `folder' From ec97a38584a9e9bdcf2b4206424611c8d278260b Mon Sep 17 00:00:00 2001 From: Denisse Orozco Date: Fri, 10 Dec 2021 07:07:31 -0500 Subject: [PATCH 20/20] ietestform: update ieutil_ commands --- src/ado_files/ietestform.ado | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/ado_files/ietestform.ado b/src/ado_files/ietestform.ado index 76862e7a..8fd4ea33 100644 --- a/src/ado_files/ietestform.ado +++ b/src/ado_files/ietestform.ado @@ -10,10 +10,14 @@ qui{ syntax [using/] , Reportsave(string) [Surveyform(string) STATAlanguage(string) date replace] + /*********************************************** Test input ***********************************************/ + * ieutil commands + qui do "${GitHub}\iefieldkit\src\ado_files/iefieldkit_aux.do" + /********* Test survey file input *********/ @@ -29,18 +33,13 @@ qui{ } local surveyform `"`using'`surveyform'"' - *Get the foldername, filename and the file extension for the survey file input - ieaux_filename using `surveyform' - - * Standarize path - local surveyform "`r(using)'" - *Test that the folder exists - ieaux_folderpath, folderpath("`r(folder)'") description("in surveyform ") + *Test that the folder exists + ieutil_folderpath using `surveyform', description("in surveyform ") * Test if the form file is xls or xlsx - ieaux_fileext using `surveyform', fileext(xlsx xls) testfileext("`r(fileext)'") - local surveyform "`r(using)'" + ieutil_fileext using `surveyform', allowed_exts(.xlsx .xls) default_ext(.xlsx) + local surveyform "`r(file_path)'" *Test if the form file exists cap confirm file "`surveyform'" @@ -55,18 +54,13 @@ qui{ *********/ ********* - *Get the foldername, filename and the file extension for the report file - ieaux_filename using `using' - - * Standarize path - local using "`r(using)'" *Test that the folder for the report file exists - ieaux_folderpath, folderpath("`r(folder)'") description("in [`reportsave'] ") + ieutil_folderpath using `using', description("in [`reportsave'] ") *Test if the file extension type from the report file is csv - ieaux_fileext `using', fileext(csv) testfileext(`r_fileext') - local using "`r(using)'" + ieutil_fileext `using', allowed_exts(.csv) default_ext(.csv) + local using "`r(file_path)'" *Tempfile that will be used to write the report tempfile report_tempfile