diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml index 5094c4490..f9e253265 100644 --- a/.github/workflows/R-CMD-check-wsl.yaml +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -54,14 +54,14 @@ jobs: run: | remotes::install_deps(dependencies = TRUE) remotes::install_cran("rcmdcheck") - remotes::install_local(path = ".") + remotes::install_local(path = ".", INSTALL_opts = "--no-test-load") install.packages("curl") shell: Rscript {0} - uses: Vampire/setup-wsl@v1 with: distribution: Ubuntu-22.04 - use-cache: 'true' + use-cache: 'false' set-as-default: 'true' - name: Install WSL Dependencies run: | @@ -74,6 +74,7 @@ jobs: - name: Install cmdstan run: | + cmdstanr::check_cmdstan_toolchain(fix = TRUE) cmdstanr::install_cmdstan(cores = 2, wsl = TRUE, overwrite = TRUE) shell: Rscript {0} diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index c3dcb77cd..28b780b5c 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -79,6 +79,7 @@ jobs: - name: Install dependencies run: | + Sys.setenv("MAKEFLAGS"="-j2") remotes::install_deps(dependencies = TRUE) remotes::install_cran("rcmdcheck") remotes::install_local(path = ".") diff --git a/.github/workflows/Test-coverage.yaml b/.github/workflows/Test-coverage.yaml index d7814eb7c..4ef2dab5b 100644 --- a/.github/workflows/Test-coverage.yaml +++ b/.github/workflows/Test-coverage.yaml @@ -59,7 +59,7 @@ jobs: - name: Install dependencies run: | install.packages(c("remotes", "curl"), dependencies = TRUE) - remotes::install_local(path = ".") + remotes::install_local(path = ".", INSTALL_opts = "--no-test-load") remotes::install_deps(dependencies = TRUE) remotes::install_cran("covr") remotes::install_cran("gridExtra") diff --git a/.github/workflows/cmdstan-tarball-check.yaml b/.github/workflows/cmdstan-tarball-check.yaml index 5bccc5206..66880675c 100644 --- a/.github/workflows/cmdstan-tarball-check.yaml +++ b/.github/workflows/cmdstan-tarball-check.yaml @@ -66,7 +66,7 @@ jobs: run: | remotes::install_deps(dependencies = TRUE) remotes::install_cran("rcmdcheck") - remotes::install_local(path = ".") + remotes::install_local(path = ".", INSTALL_opts = "--no-test-load") cmdstanr::check_cmdstan_toolchain(fix = TRUE) if (Sys.getenv("CMDSTAN_TEST_TARBALL_URL") == "latest") { cmdstanr::install_cmdstan(cores = 2, overwrite = TRUE) diff --git a/R/args.R b/R/args.R index 879f17873..74a864661 100644 --- a/R/args.R +++ b/R/args.R @@ -57,10 +57,20 @@ CmdStanArgs <- R6::R6Class( self$save_latent_dynamics <- save_latent_dynamics self$using_tempdir <- is.null(output_dir) self$model_variables <- model_variables - if (getRversion() < "3.5.0") { + if (os_is_wsl()) { + # Want to ensure that any files under WSL are written to a tempdir within + # WSL to avoid IO performance issues + self$output_dir <- ifelse(is.null(output_dir), + file.path(wsl_dir_prefix(), wsl_tempdir()), + wsl_safe_path(output_dir)) + } else if (getRversion() < "3.5.0") { self$output_dir <- output_dir %||% tempdir() } else { - self$output_dir <- output_dir %||% tempdir(check = TRUE) + if (getRversion() < "3.5.0") { + self$output_dir <- output_dir %||% tempdir() + } else { + self$output_dir <- output_dir %||% tempdir(check = TRUE) + } } self$output_dir <- repair_path(self$output_dir) self$output_basename <- output_basename @@ -525,8 +535,7 @@ DiagnoseArgs <- R6::R6Class( #' @return `TRUE` invisibly unless an error is thrown. validate_cmdstan_args <- function(self) { validate_exe_file(self$exe_file) - - checkmate::assert_directory_exists(self$output_dir, access = "rw") + assert_dir_exists(self$output_dir, access = "rw") # at least 1 run id (chain id) checkmate::assert_integerish(self$proc_ids, @@ -545,7 +554,7 @@ validate_cmdstan_args <- function(self) { self$refresh <- as.integer(self$refresh) } if (!is.null(self$data_file)) { - checkmate::assert_file_exists(self$data_file, access = "r") + assert_file_exists(self$data_file, access = "r") } num_procs <- length(self$proc_ids) validate_init(self$init, num_procs) @@ -698,7 +707,7 @@ validate_optimize_args <- function(self) { #' @return `TRUE` invisibly unless an error is thrown. validate_generate_quantities_args <- function(self) { if (!is.null(self$fitted_params)) { - checkmate::assert_file_exists(self$fitted_params, access = "r") + assert_file_exists(self$fitted_params, access = "r") } invisible(TRUE) @@ -895,7 +904,7 @@ validate_init <- function(init, num_procs) { "length 1 or number of chains.", call. = FALSE) } - checkmate::assert_file_exists(init, access = "r") + assert_file_exists(init, access = "r") } invisible(TRUE) @@ -983,7 +992,7 @@ validate_metric_file <- function(metric_file, num_procs) { return(invisible(TRUE)) } - checkmate::assert_file_exists(metric_file, access = "r") + assert_file_exists(metric_file, access = "r") if (length(metric_file) != 1 && length(metric_file) != num_procs) { stop(length(metric_file), " metric(s) provided. Must provide ", diff --git a/R/csv.R b/R/csv.R index a1bb466dc..4eff55c72 100644 --- a/R/csv.R +++ b/R/csv.R @@ -125,7 +125,7 @@ read_cmdstan_csv <- function(files, sampler_diagnostics = NULL, format = getOption("cmdstanr_draws_format", NULL)) { format <- assert_valid_draws_format(format) - checkmate::assert_file_exists(files, access = "r", extension = "csv") + assert_file_exists(files, access = "r", extension = "csv") metadata <- NULL warmup_draws <- list() draws <- list() @@ -237,7 +237,7 @@ read_cmdstan_csv <- function(files, fread_cmd <- paste0( grep_path_quotes, " -v \"^#\" --color=never \"", - output_file, + wsl_safe_path(output_file, revert = TRUE), "\"" ) } else { @@ -556,7 +556,7 @@ for (method in unavailable_methods_CmdStanFit_CSV) { #' mass matrix (or its diagonal depending on the metric). #' read_csv_metadata <- function(csv_file) { - checkmate::assert_file_exists(csv_file, access = "r", extension = "csv") + assert_file_exists(csv_file, access = "r", extension = "csv") inv_metric_next <- FALSE csv_file_info <- list() csv_file_info$inv_metric <- NULL @@ -579,7 +579,7 @@ read_csv_metadata <- function(csv_file) { fread_cmd <- paste0( grep_path_quotes, " \"^[#a-zA-Z]\" --color=never \"", - csv_file, + wsl_safe_path(csv_file, revert = TRUE), "\"" ) } else { diff --git a/R/fit.R b/R/fit.R index bb56d3b04..e46901203 100644 --- a/R/fit.R +++ b/R/fit.R @@ -19,6 +19,10 @@ CmdStanFit <- R6::R6Class( if (!is.null(private$model_methods_env_$model_ptr)) { initialize_model_pointer(private$model_methods_env_, self$data_file(), 0) } + # Need to update the output directory path to one that can be accessed + # from Windows, for the post-processing of results + self$runset$args$output_dir <- wsl_safe_path(self$runset$args$output_dir, + revert = TRUE) invisible(self) }, num_procs = function() { @@ -303,6 +307,11 @@ CmdStanFit$set("public", name = "init", value = init) #' } #' init_model_methods <- function(seed = 0, verbose = FALSE, hessian = FALSE) { + if (os_is_wsl()) { + stop("Additional model methods are not currently available with ", + "WSL CmdStan and will not be compiled", + call. = FALSE) + } require_suggested_package("Rcpp") require_suggested_package("RcppEigen") if (length(private$model_methods_env_$hpp_code_) == 0) { diff --git a/R/install.R b/R/install.R index 9909df454..7820881c3 100644 --- a/R/install.R +++ b/R/install.R @@ -93,8 +93,10 @@ install_cmdstan <- function(dir = NULL, call. = FALSE) wsl <- FALSE } else { - Sys.setenv("CMDSTANR_USE_WSL" = 1) + .cmdstanr$WSL <- TRUE } + } else { + .cmdstanr$WSL <- FALSE } if (check_toolchain) { check_cmdstan_toolchain(fix = FALSE, quiet = quiet) @@ -108,13 +110,13 @@ install_cmdstan <- function(dir = NULL, } } if (is.null(dir)) { - dir <- cmdstan_default_install_path() + dir <- cmdstan_default_install_path(wsl = wsl) if (!dir.exists(dir)) { dir.create(dir, recursive = TRUE) } } else { dir <- repair_path(dir) - checkmate::assert_directory_exists(dir, access = "rwx") + assert_dir_exists(dir, access = "rwx") } if (!is.null(version)) { if (!is.null(release_url)) { @@ -125,7 +127,6 @@ install_cmdstan <- function(dir = NULL, release_url <- paste0("https://github.com/stan-dev/cmdstan/releases/download/v", version, "/cmdstan-", version, cmdstan_arch_suffix(version), ".tar.gz") } - wsl_prefix <- ifelse(isTRUE(wsl), "wsl-", "") if (!is.null(release_url)) { if (!endsWith(release_url, ".tar.gz")) { stop(release_url, " is not a .tar.gz archive!", @@ -137,14 +138,14 @@ install_cmdstan <- function(dir = NULL, tar_name <- utils::tail(split_url[[1]], n = 1) cmdstan_ver <- substr(tar_name, 0, nchar(tar_name) - 7) tar_gz_file <- paste0(cmdstan_ver, ".tar.gz") - dir_cmdstan <- file.path(dir, paste0(wsl_prefix, cmdstan_ver)) + dir_cmdstan <- file.path(dir, cmdstan_ver) dest_file <- file.path(dir, tar_gz_file) } else { ver <- latest_released_version() message("* Latest CmdStan release is v", ver) cmdstan_ver <- paste0("cmdstan-", ver, cmdstan_arch_suffix(ver)) tar_gz_file <- paste0(cmdstan_ver, ".tar.gz") - dir_cmdstan <- file.path(dir, paste0(wsl_prefix, cmdstan_ver)) + dir_cmdstan <- file.path(dir, cmdstan_ver) message("* Installing CmdStan v", ver, " in ", dir_cmdstan) message("* Downloading ", tar_gz_file, " from GitHub...") download_url <- github_download_url(ver) @@ -164,17 +165,34 @@ install_cmdstan <- function(dir = NULL, stop("Download of CmdStan failed. Please try again.", call. = FALSE) } message("* Download complete") - message("* Unpacking archive...") - untar_rc <- utils::untar( - dest_file, - exdir = dir_cmdstan, - extras = "--strip-components 1" - ) - if (untar_rc != 0) { - stop("Problem extracting tarball. Exited with return code: ", untar_rc, call. = FALSE) + if (wsl) { + # Significantly faster to use WSL to untar the downloaded archive, as there are + # similar IO issues accessing the WSL filesystem from windows + wsl_tar_gz_file <- gsub(paste0("//wsl$/", wsl_distro_name()), "", + dest_file, fixed = TRUE) + wsl_tar_gz_file <- wsl_safe_path(wsl_tar_gz_file) + untar_rc <- processx::run( + command = "wsl", + args = c("tar", "-xf", wsl_tar_gz_file, "-C", + gsub(tar_gz_file, "", wsl_tar_gz_file)) + ) + remove_rc <- processx::run( + command = "wsl", + args = c("rm", wsl_tar_gz_file) + ) + } else { + untar_rc <- utils::untar( + dest_file, + exdir = dir_cmdstan, + extras = "--strip-components 1" + ) + if (untar_rc != 0) { + stop("Problem extracting tarball. Exited with return code: ", untar_rc, call. = FALSE) + } + file.remove(dest_file) } - file.remove(dest_file) + cmdstan_make_local(dir = dir_cmdstan, cpp_options = cpp_options, append = TRUE) # Setting up native M1 compilation of CmdStan and its downstream libraries if (is_rosetta2()) { @@ -186,7 +204,7 @@ install_cmdstan <- function(dir = NULL, append = TRUE ) } - if (is_rtools42_toolchain() && !os_is_wsl()) { + if (is_rtools42_toolchain() && !wsl) { cmdstan_make_local( dir = dir_cmdstan, cpp_options = list( @@ -521,10 +539,7 @@ install_toolchain <- function(quiet = FALSE) { } check_wsl_toolchain <- function() { - wsl_inaccessible <- processx::run(command = "wsl", - args = "uname", - error_on_status = FALSE) - if (wsl_inaccessible$status) { + if (!wsl_installed()) { stop("\n", "A WSL distribution is not installed or is not accessible.", "\n", "Please see the Microsoft documentation for guidance on installing WSL: ", "\n", "https://docs.microsoft.com/en-us/windows/wsl/install", diff --git a/R/model.R b/R/model.R index a1bfddc64..9deb62c43 100644 --- a/R/model.R +++ b/R/model.R @@ -229,7 +229,7 @@ CmdStanModel <- R6::R6Class( self$functions <- new.env() self$functions$compiled <- FALSE if (!is.null(stan_file)) { - checkmate::assert_file_exists(stan_file, access = "r", extension = "stan") + assert_file_exists(stan_file, access = "r", extension = "stan") checkmate::assert_flag(compile) private$stan_file_ <- absolute_path(stan_file) private$stan_code_ <- readLines(stan_file) @@ -250,7 +250,7 @@ CmdStanModel <- R6::R6Class( ext <- if (os_is_windows() && !os_is_wsl()) "exe" else "" private$exe_file_ <- repair_path(absolute_path(exe_file)) if (is.null(stan_file)) { - checkmate::assert_file_exists(private$exe_file_, access = "r", extension = ext) + assert_file_exists(private$exe_file_, access = "r", extension = ext) private$model_name_ <- sub(" ", "_", strip_ext(basename(private$exe_file_))) } } @@ -317,7 +317,7 @@ CmdStanModel <- R6::R6Class( if (is.null(dir)) { dir <- dirname(private$stan_file_) } - checkmate::assert_directory_exists(dir, access = "r") + assert_dir_exists(dir, access = "r") new_hpp_loc <- file.path(dir, paste0(strip_ext(basename(private$stan_file_)), ".hpp")) file.copy(self$hpp_file(), new_hpp_loc, overwrite = TRUE) file.remove(self$hpp_file()) @@ -471,7 +471,7 @@ compile <- function(quiet = TRUE, } if (!is.null(dir)) { dir <- repair_path(dir) - checkmate::assert_directory_exists(dir, access = "rw") + assert_dir_exists(dir, access = "rw") if (length(self$exe_file()) != 0) { private$exe_file_ <- file.path(dir, basename(self$exe_file())) } @@ -524,6 +524,15 @@ compile <- function(quiet = TRUE, } } + if (os_is_wsl() && (compile_model_methods || compile_standalone)) { + warning("Additional model methods and standalone functions are not ", + "currently available with WSL CmdStan and will not be compiled", + call. = FALSE) + compile_model_methods <- FALSE + compile_standalone <- FALSE + compile_hessian_method <- FALSE + } + temp_stan_file <- tempfile(pattern = "model-", fileext = ".stan") file.copy(self$stan_file(), temp_stan_file, overwrite = TRUE) temp_file_no_ext <- strip_ext(temp_stan_file) @@ -629,6 +638,12 @@ compile <- function(quiet = TRUE, file.remove(exe) } file.copy(tmp_exe, exe, overwrite = TRUE) + if (os_is_wsl()) { + res <- processx::run( + command = "wsl", + args = c("chmod", "+x", wsl_safe_path(exe)) + ) + } private$exe_file_ <- exe private$cpp_options_ <- cpp_options private$precompile_cpp_options_ <- NULL @@ -1806,7 +1821,7 @@ cpp_options_to_compile_flags <- function(cpp_options) { include_paths_stanc3_args <- function(include_paths = NULL) { stancflags <- NULL if (!is.null(include_paths)) { - checkmate::assert_directory_exists(include_paths, access = "r") + assert_dir_exists(include_paths, access = "r") include_paths <- sapply(absolute_path(include_paths), wsl_safe_path) paths_w_space <- grep(" ", include_paths) include_paths[paths_w_space] <- paste0("'", include_paths[paths_w_space], "'") diff --git a/R/path.R b/R/path.R index a7a2ce252..a887a6174 100644 --- a/R/path.R +++ b/R/path.R @@ -45,7 +45,7 @@ set_cmdstan_path <- function(path = NULL) { path <- absolute_path(path) .cmdstanr$PATH <- path .cmdstanr$VERSION <- read_cmdstan_version(path) - .cmdstanr$WSL <- grepl("wsl-cmdstan", path) + .cmdstanr$WSL <- grepl("//wsl$", path, fixed = TRUE) message("CmdStan path set to: ", path) } else { warning("Path not set. Can't find directory: ", path, call. = FALSE) @@ -107,13 +107,18 @@ stop_no_path <- function() { #' @keywords internal #' @param old Should the old default path (.cmdstanr) be used instead of the new #' one (.cmdstan)? Defaults to `FALSE` and may be removed in a future release. +#' @param wsl Return the directory for WSL installations? #' @return The installation path. #' @export -cmdstan_default_install_path <- function(old = FALSE) { - if (old) { - file.path(Sys.getenv("HOME"), ".cmdstanr") +cmdstan_default_install_path <- function(old = FALSE, wsl = FALSE) { + if (wsl) { + file.path(paste0(wsl_dir_prefix(wsl = TRUE), wsl_home_dir()), ".cmdstan") } else { - file.path(Sys.getenv("HOME"), ".cmdstan") + if (old) { + file.path(Sys.getenv("HOME"), ".cmdstanr") + } else { + file.path(Sys.getenv("HOME"), ".cmdstan") + } } } @@ -140,47 +145,54 @@ cmdstan_default_path <- function(old = FALSE, dir = NULL) { } else { installs_path <- cmdstan_default_install_path(old) } - if (dir.exists(installs_path)) { - cmdstan_installs <- list.dirs(path = installs_path, recursive = FALSE, full.names = FALSE) - # if installed in cmdstan folder with no version move to cmdstan-version folder - if ("cmdstan" %in% cmdstan_installs) { - ver <- read_cmdstan_version(file.path(installs_path, "cmdstan")) - old_path <- file.path(installs_path, "cmdstan") - new_path <- file.path(installs_path, paste0("cmdstan-", ver)) - file.rename(old_path, new_path) - cmdstan_installs <- list.dirs(path = installs_path, recursive = FALSE, full.names = FALSE) - } - if (length(cmdstan_installs) > 0) { - wsl_installs <- grep("^wsl-cmdstan-", cmdstan_installs, value = TRUE) - cmdstan_installs <- cmdstan_installs[!grepl("wsl-", cmdstan_installs)] - cmdstan_installs <- grep("^cmdstan-", cmdstan_installs, value = TRUE) - if (length(wsl_installs) > 0) { - wsl_installs_trim <- gsub("wsl-", "", wsl_installs, fixed = TRUE) - wsl_latest <- sort(wsl_installs_trim, decreasing = TRUE)[1] - if (length(cmdstan_installs) > 0) { - non_wsl_latest <- sort(cmdstan_installs, decreasing = TRUE)[1] - latest_cmdstan <- ifelse(wsl_latest > non_wsl_latest - || wsl_latest == non_wsl_latest, - grep(wsl_latest, wsl_installs, value = TRUE), - non_wsl_latest) - } else { - latest_cmdstan <- grep(wsl_latest, wsl_installs, value = TRUE) - } - } else { - latest_cmdstan <- sort(cmdstan_installs, decreasing = TRUE)[1] - } - if (is_release_candidate(latest_cmdstan)) { - non_rc_path <- strsplit(latest_cmdstan, "-rc")[[1]][1] - if (dir.exists(file.path(installs_path, non_rc_path))) { - latest_cmdstan <- non_rc_path - } - } + wsl_installed <- wsl_installed() + if (!isTRUE(wsl_installed)) { + wsl_installs_path <- NULL + wsl_path_exists <- FALSE + } else { + wsl_installs_path <- cmdstan_default_install_path(old, wsl = TRUE) + wsl_path_linux <- gsub(wsl_dir_prefix(wsl = TRUE), "", wsl_installs_path, + fixed=TRUE) + wsl_path_exists <- isTRUE(.wsl_check_exists(wsl_path_linux)) + } + if (dir.exists(installs_path) || wsl_path_exists) { + latest_cmdstan <- ifelse(dir.exists(installs_path), + .latest_cmdstan_installed(installs_path), "") + latest_wsl_cmdstan <- ifelse(wsl_path_exists, + .latest_cmdstan_installed(wsl_installs_path), "") + if (latest_wsl_cmdstan >= latest_cmdstan) { + return(file.path(wsl_installs_path, latest_wsl_cmdstan)) + } else { return(file.path(installs_path, latest_cmdstan)) } } NULL } +.latest_cmdstan_installed <- function(installs_path) { + cmdstan_installs <- list.dirs(path = installs_path, recursive = FALSE, full.names = FALSE) + # if installed in cmdstan folder with no version move to cmdstan-version folder + if ("cmdstan" %in% cmdstan_installs) { + ver <- read_cmdstan_version(file.path(installs_path, "cmdstan")) + old_path <- file.path(installs_path, "cmdstan") + new_path <- file.path(installs_path, paste0("cmdstan-", ver)) + file.rename(old_path, new_path) + cmdstan_installs <- list.dirs(path = installs_path, recursive = FALSE, full.names = FALSE) + } + latest_cmdstan <- "" + if (length(cmdstan_installs) > 0) { + cmdstan_installs <- grep("^cmdstan-", cmdstan_installs, value = TRUE) + latest_cmdstan <- sort(cmdstan_installs, decreasing = TRUE)[1] + if (is_release_candidate(latest_cmdstan)) { + non_rc_path <- strsplit(latest_cmdstan, "-rc")[[1]][1] + if (dir.exists(file.path(installs_path, non_rc_path))) { + latest_cmdstan <- non_rc_path + } + } + } + latest_cmdstan +} + #' Find the version of CmdStan from makefile #' @noRd diff --git a/R/run.R b/R/run.R index ec5f9410d..d17d4acb6 100644 --- a/R/run.R +++ b/R/run.R @@ -29,6 +29,16 @@ CmdStanRun <- R6::R6Class( if (self$args$save_latent_dynamics) { private$latent_dynamics_files_ <- self$new_latent_dynamics_files() } + if (os_is_wsl()) { + # While the executable built under WSL will be stored in the Windows + # filesystem alongside the model code, we place a copy in a WSL temp + # directory prior to execution to avoid IO perfomance impacts + wsl_tmpdir <- wsl_tempdir() + file.copy(from = args$exe_file, + to = file.path(wsl_dir_prefix(), wsl_tmpdir)) + args$exe_file <- file.path(wsl_tmpdir, basename(args$exe_file)) + processx::run("wsl", args = c("chmod", "+x", args$exe_file)) + } invisible(self) }, num_procs = function() self$procs$num_procs(), diff --git a/R/utils.R b/R/utils.R index b1057e07b..31f56f3d0 100644 --- a/R/utils.R +++ b/R/utils.R @@ -99,7 +99,8 @@ repair_path <- function(path) { } path <- path.expand(path) path <- gsub("\\\\", "/", path) - path <- gsub("//", "/", path) + # WSL cmdstan path is a network path and needs the leading // + path <- gsub("//(?!wsl)", "/", path, perl = TRUE) if (endsWith(path, "/")) { # remove trailing "/" path <- substr(path, 1, nchar(path) - 1) @@ -142,63 +143,6 @@ strip_ext <- function(file) { } absolute_path <- Vectorize(.absolute_path, USE.NAMES = FALSE) -# When providing the model path to WSL, it needs to be in reference to the -# to Windows mount point (/mnt/drive-letter) within the WSL install: -# e.g., C:/Users/... -> /mnt/c/Users/... -wsl_safe_path <- function(path = NULL, revert = FALSE) { - if (!is.character(path) || is.null(path) || !os_is_wsl()) { - return(path) - } - if (revert) { - if (!grepl("^/mnt/", path)) { - return(path) - } - strip_mnt <- gsub("^/mnt/", "", path) - drive_letter <- strtrim(strip_mnt, 1) - path <- gsub(paste0("^/mnt/", drive_letter), - paste0(toupper(drive_letter), ":"), - path) - } else { - path_already_safe <- grepl("^/mnt/", path) - if (os_is_wsl() && !isTRUE(path_already_safe) && !is.na(path)) { - base_file <- basename(path) - path <- dirname(path) - abs_path <- repair_path(utils::shortPathName(path)) - drive_letter <- tolower(strtrim(abs_path, 1)) - path <- gsub(paste0(drive_letter, ":"), - paste0("/mnt/", drive_letter), - abs_path, - ignore.case = TRUE) - path <- paste0(path, "/", base_file) - } - } - path -} - -# Running commands through WSL requires using 'wsl' as the command with the -# intended command (e.g., stanc) as the first argument. This function acts as -# a wrapper around processx::run() to apply this change where necessary, and -# forward all other arguments -wsl_compatible_run <- function(...) { - run_args <- list(...) - if (os_is_wsl()) { - command <- run_args$command - run_args$command <- "wsl" - run_args$args <- c(command, run_args$args) - } - do.call(processx::run, run_args) -} - -wsl_compatible_process_new <- function(...) { - run_args <- list(...) - if (os_is_wsl()) { - command <- run_args$command - run_args$command <- "wsl" - run_args$args <- c(command, run_args$args) - } - do.call(processx::process$new, run_args) -} - # read, write, and copy files -------------------------------------------- #' Copy temporary files (e.g., output, data) to a different location @@ -224,7 +168,7 @@ copy_temp_files <- timestamp = TRUE, random = TRUE, ext = ".csv") { - checkmate::assert_directory_exists(new_dir, access = "w") + assert_dir_exists(new_dir, access = "w") destinations <- generate_file_names( basename = new_basename, ext = ext, @@ -513,6 +457,207 @@ as_mcmc.list <- function(x) { return(mcmc_list) } +# WSL-related helper functions ------------------------------------------ + +# When providing the model path to WSL, it needs to be in reference to the +# to Windows mount point (/mnt/drive-letter) within the WSL install: +# e.g., C:/Users/... -> /mnt/c/Users/... +wsl_safe_path <- function(path = NULL, revert = FALSE) { + if (!is.character(path) || is.null(path) || !os_is_wsl()) { + return(path) + } + if (revert) { + if (!grepl("^/mnt/", path)) { + return(path) + } + strip_mnt <- gsub("^/mnt/", "", path) + drive_letter <- strtrim(strip_mnt, 1) + path <- gsub(paste0("^/mnt/", drive_letter), + paste0(toupper(drive_letter), ":"), + path) + } else if (grepl("^//wsl", path)) { + path <- gsub(wsl_dir_prefix(), "", path, fixed = TRUE) + } else { + path_already_safe <- grepl("^/mnt/", path) + if (os_is_wsl() && !isTRUE(path_already_safe) && !is.na(path)) { + base_file <- basename(path) + path <- dirname(path) + abs_path <- repair_path(utils::shortPathName(path)) + drive_letter <- tolower(strtrim(abs_path, 1)) + path <- gsub(paste0(drive_letter, ":"), + paste0("/mnt/", drive_letter), + abs_path, + ignore.case = TRUE) + path <- paste0(path, "/", base_file) + } + } + path +} + +# Running commands through WSL requires using 'wsl' as the command with the +# intended command (e.g., stanc) as the first argument. This function acts as +# a wrapper around processx::run() to apply this change where necessary, and +# forward all other arguments +wsl_compatible_run <- function(...) { + run_args <- list(...) + if (os_is_wsl()) { + command <- run_args$command + run_args$command <- "wsl" + if (!is.null(run_args$wd)) { + wd <- wsl_safe_path(run_args$wd) + run_args$wd <- NULL + run_args$args <- c(c("cd", wd, "&&"), command, run_args$args) + } else { + run_args$args <- c(command, run_args$args) + } + } + do.call(processx::run, run_args) +} + +wsl_compatible_process_new <- function(...) { + run_args <- list(...) + if (os_is_wsl()) { + command <- run_args$command + run_args$command <- "wsl" + if (!is.null(run_args$wd)) { + wd <- wsl_safe_path(run_args$wd) + run_args$wd <- NULL + run_args$args <- c(c("cd", wd, "&&"), command, run_args$args) + } else { + run_args$args <- c(command, run_args$args) + } + } + do.call(processx::process$new, run_args) +} + +wsl_installed <- function() { + tryCatch({ + # Call can hang indefinitely on Github actions, so explicitly kill + p <- processx::process$new("wsl", "uname") + Sys.sleep(5) + if (p$is_alive()) { + p$kill() + FALSE + } else { + status <- p$get_exit_status() + if (is.null(status)) { + FALSE + } + isTRUE(status == 0) + } + }, error = function(e) { FALSE }, finally = function(ret) { ret }) +} + +wsl_distro_name <- function() { + name <- processx::run( + command = "wsl", + args = c("echo", "$WSL_DISTRO_NAME") + )$stdout + gsub("\n", "", name, fixed = TRUE) +} + +wsl_home_dir <- function() { + dir <- processx::run( + command = "wsl", + args = c("echo", "$HOME") + )$stdout + gsub("\n", "", dir, fixed = TRUE) +} + +wsl_dir_prefix <- function(wsl = FALSE) { + if (os_is_wsl() || wsl) { + paste0("//wsl$/", wsl_distro_name()) + } else { + "" + } +} + +wsl_tempdir <- function() { + dir <- processx::run(command = "wsl", + args = c("mktemp", "-d"))$stdout + gsub("\n", "", dir, fixed = TRUE) +} + +# The checkmate file and directory assertion functions don't register the WSL +# network path as legitimate, and will always error. To avoid this we create a +# new checking functions with WSL handling, and then pass these to +# checkmate::makeAssertionFunction to replicate the existing assertion functionality +check_dir_exists <- function(dir, access = NULL) { + if (os_is_wsl()) { + if (!checkmate::qtest(dir, "S+")) { + return("No directory provided.") + } + checks <- sapply(dir, .wsl_check_exists, is_dir = TRUE, access = access) + if (any(as.character(checks) != "TRUE")) { + grep("TRUE", checks, value = TRUE, invert = TRUE)[1] + } else { + TRUE + } + } else { + checkmate::checkDirectoryExists(dir, access = access) + } +} + +check_file_exists <- function(files, access = NULL, ...) { + if (os_is_wsl()) { + if (!checkmate::qtest(files, "S+")) { + return("No file provided.") + } + checks <- sapply(files, .wsl_check_exists, is_dir = FALSE, access = access) + if (any(as.character(checks) != "TRUE")) { + grep("TRUE", checks, value = TRUE, invert = TRUE)[1] + } else { + TRUE + } + } else { + checkmate::checkFileExists(files, access = access, ...) + } +} + +.wsl_check_exists <- function(path, is_dir = TRUE, access = NULL) { + if (!wsl_installed()) { + return(FALSE) + } + path_check <- processx::run( + command = "wsl", + args = c("ls", "-la", wsl_safe_path(path)), + error_on_status = FALSE + ) + + if (path_check$status != 0) { + path <- gsub("^./", "", path) + err <- ifelse(is_dir, + paste0("Directory '", path, "' does not exist"), + paste0("File does not exist: '", path, "'")) + return(err) + } + + path_metadata <- strsplit(path_check$stdout, split = "\n", + fixed = TRUE)[[1]] + + wsl_user <- processx::run( + command = "wsl", + args = c("echo", "$USER"), + error_on_status = FALSE + )$stdout + wsl_user <- gsub("\n", "", wsl_user, fixed = TRUE) + + path_metadata <- grep(wsl_user, path_metadata, value = TRUE) + + if (!is.null(access)) { + path_permissions <- strsplit(path_metadata, " ", fixed = TRUE)[[1]][1] + if (!any(grepl(access, path_permissions))) { + name <- ifelse(is_dir, "directory", "file") + return(paste0("Specified ", name, ": ", path, + " does not have access permission ", access)) + } + } + TRUE +} + +assert_dir_exists <- checkmate::makeAssertionFunction(check_dir_exists) +assert_file_exists <- checkmate::makeAssertionFunction(check_file_exists) + # Model methods & expose_functions helpers ------------------------------------------------------ get_cmdstan_flags <- function(flag_name) { cmdstan_path <- cmdstanr::cmdstan_path() @@ -711,6 +856,11 @@ compile_functions <- function(env, verbose = FALSE, global = FALSE) { } expose_functions <- function(function_env, global = FALSE, verbose = FALSE) { + if (os_is_wsl()) { + stop("Standalone functions are not currently available with ", + "WSL CmdStan and will not be compiled", + call. = FALSE) + } require_suggested_package("Rcpp") require_suggested_package("RcppEigen") require_suggested_package("decor") @@ -731,4 +881,4 @@ expose_functions <- function(function_env, global = FALSE, verbose = FALSE) { compile_functions(function_env, verbose, global) } invisible(NULL) -} \ No newline at end of file +} diff --git a/man/cmdstan_default_install_path.Rd b/man/cmdstan_default_install_path.Rd index 803d82a1b..07ddb8377 100644 --- a/man/cmdstan_default_install_path.Rd +++ b/man/cmdstan_default_install_path.Rd @@ -4,11 +4,13 @@ \alias{cmdstan_default_install_path} \title{cmdstan_default_install_path} \usage{ -cmdstan_default_install_path(old = FALSE) +cmdstan_default_install_path(old = FALSE, wsl = FALSE) } \arguments{ \item{old}{Should the old default path (.cmdstanr) be used instead of the new one (.cmdstan)? Defaults to \code{FALSE} and may be removed in a future release.} + +\item{wsl}{Return the directory for WSL installations?} } \value{ The installation path. diff --git a/man/cmdstanr-package.Rd b/man/cmdstanr-package.Rd index b093d1fbb..21e83186b 100644 --- a/man/cmdstanr-package.Rd +++ b/man/cmdstanr-package.Rd @@ -33,8 +33,8 @@ algorithms, and writing results to output files. \subsection{Advantages of RStan}{ \itemize{ -\item Advanced features. We are working on making these available outside of -RStan but currently they are only available to R users via RStan: +\item Advanced features. We are working on making these available outside +of RStan but currently they are only available to R users via RStan: \itemize{ \item \code{rstan::log_prob()} \item \code{rstan::grad_log_prob()} @@ -47,13 +47,14 @@ Stan programs (like \strong{rstanarm}) on CRAN. \subsection{Advantages of CmdStanR}{ \itemize{ -\item Compatible with latest versions of Stan. Keeping up with Stan releases -is complicated for RStan, often requiring non-trivial changes to the -\strong{rstan} package and new CRAN releases of both \strong{rstan} and -\strong{StanHeaders}. With CmdStanR the latest improvements in Stan will be -available from R immediately after updating CmdStan using -\code{cmdstanr::install_cmdstan()}. -\item Fewer installation issues (e.g., no need to mess with Makevars files). +\item Compatible with latest versions of Stan. Keeping up with Stan +releases is complicated for RStan, often requiring non-trivial +changes to the \strong{rstan} package and new CRAN releases of both +\strong{rstan} and \strong{StanHeaders}. With CmdStanR the latest improvements +in Stan will be available from R immediately after updating CmdStan +using \code{cmdstanr::install_cmdstan()}. +\item Fewer installation issues (e.g., no need to mess with Makevars +files). \item Running Stan via external processes results in fewer unexpected crashes, especially in RStudio. \item Less memory overhead. diff --git a/tests/testthat/test-fit-shared.R b/tests/testthat/test-fit-shared.R index f41c9344c..921a8d8cc 100644 --- a/tests/testthat/test-fit-shared.R +++ b/tests/testthat/test-fit-shared.R @@ -452,11 +452,8 @@ test_that("sampling with inits works with include_paths", { file.remove(exe) } - expect_interactive_message( - mod_w_include <- cmdstan_model(stan_file = stan_program_w_include, quiet = TRUE, - include_paths = test_path("resources", "stan")), - "Compiling Stan program" - ) + mod_w_include <- cmdstan_model(stan_file = stan_program_w_include, quiet = FALSE, + include_paths = test_path("resources", "stan")) data_list <- list(N = 10, y = c(0,1,0,0,0,0,0,0,0,1)) diff --git a/tests/testthat/test-install.R b/tests/testthat/test-install.R index 39192033d..2d312e8a6 100644 --- a/tests/testthat/test-install.R +++ b/tests/testthat/test-install.R @@ -1,7 +1,5 @@ context("install") -wsl_prefix <- ifelse(os_is_wsl(), "wsl-", "") - cmdstan_test_tarball_url <- Sys.getenv("CMDSTAN_TEST_TARBALL_URL") if (!nzchar(cmdstan_test_tarball_url)) { cmdstan_test_tarball_url <- NULL @@ -28,13 +26,13 @@ test_that("install_cmdstan() successfully installs cmdstan", { test_that("install_cmdstan() errors if installation already exists", { install_dir <- cmdstan_default_install_path() - dir <- file.path(install_dir, paste0(wsl_prefix, "cmdstan-2.23.0")) + dir <- file.path(install_dir, "cmdstan-2.23.0") if (!dir.exists(dir)) { - dir.create(dir) + dir.create(dir, recursive = TRUE) } expect_warning( install_cmdstan(dir = install_dir, overwrite = FALSE, - version = "2.23.0", wsl = os_is_wsl()), + version = "2.23.0", wsl = FALSE), "An installation already exists", fixed = TRUE ) @@ -47,7 +45,7 @@ test_that("install_cmdstan() errors if it times out", { dir <- tempdir(check = TRUE) } ver <- latest_released_version() - dir_exists <- dir.exists(file.path(dir, paste0(wsl_prefix, "cmdstan-",ver))) + dir_exists <- dir.exists(file.path(dir, paste0("cmdstan-",ver))) # with quiet=TRUE expect_warning( expect_message( @@ -59,7 +57,7 @@ test_that("install_cmdstan() errors if it times out", { "increasing the value of the 'timeout' argument and running again with 'quiet=FALSE'", fixed = TRUE ) - dir_exists <- dir.exists(file.path(dir, paste0(wsl_prefix,"cmdstan-",ver))) + dir_exists <- dir.exists(file.path(dir, paste0("cmdstan-",ver))) # with quiet=FALSE expect_warning( expect_message( @@ -125,7 +123,7 @@ test_that("install_cmdstan() works with version and release_url", { "version and release_url shouldn't both be specified", fixed = TRUE ) - expect_true(dir.exists(file.path(dir, paste0(wsl_prefix, "cmdstan-2.27.0")))) + expect_true(dir.exists(file.path(dir, "cmdstan-2.27.0"))) set_cmdstan_path(cmdstan_default_path()) }) @@ -199,3 +197,4 @@ test_that("github_download_url constructs correct url", { "https://github.com/stan-dev/cmdstan/releases/download/vFOO/cmdstan-FOO.tar.gz" ) }) + diff --git a/tests/testthat/test-model-expose-functions.R b/tests/testthat/test-model-expose-functions.R index d1e5a820f..a16b34ee4 100644 --- a/tests/testthat/test-model-expose-functions.R +++ b/tests/testthat/test-model-expose-functions.R @@ -14,6 +14,7 @@ fit <- mod$sample(data = data_list) test_that("Functions can be exposed in model object", { + skip_if(os_is_wsl()) mod$expose_functions(verbose = TRUE) expect_equal( @@ -30,6 +31,7 @@ test_that("Functions can be exposed in model object", { }) test_that("Functions can be exposed in fit object", { + skip_if(os_is_wsl()) fit$expose_functions(verbose = TRUE) expect_equal( @@ -39,6 +41,7 @@ test_that("Functions can be exposed in fit object", { }) test_that("Compiled functions can be copied to global environment", { + skip_if(os_is_wsl()) expect_message( fit$expose_functions(global = TRUE), "Functions already compiled, copying to global environment", @@ -53,6 +56,7 @@ test_that("Compiled functions can be copied to global environment", { test_that("Functions can be compiled with model", { + skip_if(os_is_wsl()) mod <- cmdstan_model(model, force_recompile = TRUE, compile_standalone = TRUE) fit <- mod$sample(data = data_list) @@ -80,6 +84,7 @@ test_that("Functions can be compiled with model", { }) test_that("rng functions can be exposed", { + skip_if(os_is_wsl()) function_decl <- "functions { real normal_rng(real mu) { return normal_rng(mu, 1); } }" stan_prog <- paste(function_decl, paste(readLines(testing_stan_file("bernoulli")), diff --git a/tests/testthat/test-model-methods.R b/tests/testthat/test-model-methods.R index d1b6f7fe8..35feeec1e 100644 --- a/tests/testthat/test-model-methods.R +++ b/tests/testthat/test-model-methods.R @@ -6,6 +6,7 @@ data_list <- testing_data("bernoulli") fit <- mod$sample(data = data_list, chains = 1) test_that("Methods error if not compiled", { + skip_if(os_is_wsl()) expect_error( fit$log_prob(NULL), "The method has not been compiled, please call `init_model_methods()` first", @@ -34,6 +35,7 @@ test_that("Methods error if not compiled", { }) test_that("User warned about higher-order autodiff with hessian", { + skip_if(os_is_wsl()) expect_message( fit$init_model_methods(hessian = TRUE, verbose = TRUE), "The hessian method relies on higher-order autodiff which is still experimental. Please report any compilation errors that you encounter", @@ -42,6 +44,7 @@ test_that("User warned about higher-order autodiff with hessian", { }) test_that("Methods return correct values", { + skip_if(os_is_wsl()) lp <- fit$log_prob(upars=c(0.1)) expect_equal(lp, -8.6327599208828509347) @@ -80,6 +83,7 @@ test_that("Methods return correct values", { test_that("methods error for incorrect inputs", { + skip_if(os_is_wsl()) expect_error( fit$log_prob(c(1,2)), "Model has 1 unconstrained parameter(s), but 2 were provided!", @@ -116,6 +120,7 @@ test_that("methods error for incorrect inputs", { }) test_that("Methods error with already-compiled model", { + skip_if(os_is_wsl()) mod <- testing_model("bernoulli") data_list <- testing_data("bernoulli") fit <- mod$sample(data = data_list, chains = 1) @@ -127,6 +132,7 @@ test_that("Methods error with already-compiled model", { }) test_that("Methods can be compiled with model", { + skip_if(os_is_wsl()) mod <- cmdstan_model(testing_stan_file("bernoulli"), force_recompile = TRUE, compile_model_methods = TRUE, diff --git a/tests/testthat/test-model-output_dir.R b/tests/testthat/test-model-output_dir.R index c0c6bb23c..4870051cf 100644 --- a/tests/testthat/test-model-output_dir.R +++ b/tests/testthat/test-model-output_dir.R @@ -17,14 +17,19 @@ test_that("all fitting methods work with output_dir", { dir.create(method_dir) } - # no output_dir means should use tempdir - fit <- testing_fit("bernoulli", method = method, seed = 123) - expect_equal(fit$runset$args$output_dir, absolute_path(tempdir())) - + # WSL models use internal WSL tempdir + if (!os_is_wsl()) { + # no output_dir means should use tempdir + fit <- testing_fit("bernoulli", method = method, seed = 123) + expect_equal(fit$runset$args$output_dir, absolute_path(tempdir())) + } # specifying output_dir fit <- testing_fit("bernoulli", method = method, seed = 123, output_dir = method_dir) - expect_equal(fit$runset$args$output_dir, absolute_path(method_dir)) + # WSL path manipulations result in a short path which slightly differs + # from the original tempdir(), so need to normalise both for comparison + expect_equal(normalizePath(fit$runset$args$output_dir), + normalizePath(method_dir)) expect_equal(length(list.files(method_dir)), fit$num_procs()) @@ -80,6 +85,7 @@ test_that("output_dir works with trailing /", { seed = 123, output_dir = paste0(test_dir,"/") ) - expect_equal(fit$runset$args$output_dir, absolute_path(test_dir)) + expect_equal(normalizePath(fit$runset$args$output_dir), + normalizePath(test_dir)) expect_equal(length(list.files(test_dir)), fit$num_procs()) }) diff --git a/tests/testthat/test-path.R b/tests/testthat/test-path.R index daed18ae3..03d9f48d7 100644 --- a/tests/testthat/test-path.R +++ b/tests/testthat/test-path.R @@ -10,7 +10,8 @@ unset_cmdstan_path() test_that("Setting path works and confirms with message", { expect_message( set_cmdstan_path(PATH), - paste("CmdStan path set to:", PATH) + paste("CmdStan path set to:", PATH), + fixed = TRUE ) expect_equal(.cmdstanr$PATH, PATH) })