~enricoschumann/SNBdata

607ce810e6d824251b1cd5cdba34d2a30f5228ca — Enrico Schumann 1 year, 2 days ago 5fe53c8
Add functions that work for both tables and datasets
5 files changed, 514 insertions(+), 141 deletions(-)

M DESCRIPTION
M NAMESPACE
M NEWS
M R/functions.R
M man/SNBdata.Rd
M DESCRIPTION => DESCRIPTION +10 -10
@@ 1,21 1,21 @@
Package: SNBdata
Type: Package
Title: Download Data from the Swiss National Bank (SNB)
Version: 0.1.0
Date: 2023-08-31
Version: 0.2.0
Date: 2023-09-15
Maintainer: Enrico Schumann <es@enricoschumann.net>
Authors@R: person("Enrico", "Schumann",
                  role  = c("aut", "cre"),
                  email = "es@enricoschumann.net",
                  comment = c(ORCID = "0000-0001-7601-6576"))
Description: Functions for downloading data from the Swiss
    National Bank (SNB; <https://www.snb.ch/>), the
    Swiss central bank.  The package is lightweight and
    comes with few dependencies; suggested packages are
    used only if data is to be transformed into
    particular data structures, for instance into 'zoo'
    objects. Downloaded data can optionally be cached,
    to avoid repeated downloads of the same files.
Description: Functions for downloading data (tables and
     datasets) from the Swiss National Bank (SNB;
     <https://www.snb.ch/>), the Swiss central bank.  The
     package is lightweight and comes with few dependencies;
     suggested packages are used only if data is to be
     transformed into particular data structures, for instance
     into 'zoo' objects.  Downloaded data can optionally be
     cached, to avoid repeated downloads of the same files.
Suggests: jsonlite, zoo
License: GPL-3
URL: http://enricoschumann.net/R/packages/SNBdata/ ,

M NAMESPACE => NAMESPACE +5 -4
@@ 1,6 1,7 @@
importFrom("utils", "askYesNo", "download.file", "read.table", "unzip")
importFrom("utils",
           "askYesNo", "download.file", "read.table", "unzip")
export(
    "fetch_table",
    "fetch_table_date",
    "fetch_table_info"
    "fetch_data",
    "fetch_last_update",
    "fetch_info"
)

M NEWS => NEWS +8 -0
@@ 1,3 1,11 @@
v0.2.0  (2023-09-15; not released yet)

  o New functions "fetch_data", "fetch_last_update" and
    "fetch_info", which replace the previous functions
    that worked for only tables or datasets.

  o ....

v0.1.0  (2023-08-31)

  o Initial version: Allows to download tables from the

M R/functions.R => R/functions.R +456 -100
@@ 1,37 1,36 @@
fetch_table_date <- function(table = "rendoblid",
                             dest.dir = NULL,
                             verbose = TRUE,
                             language = "en", ...) {
fetch_data <- function(id,
                       type = "table",
                       dest.dir = NULL,
                       return.class = NULL,
                       verbose = TRUE,
                       method,
                       language = "en",
                       name.sep = " :: ",
                       na.drop = TRUE,
                       do.timeseries = FALSE, ...) {

    if (type == "table") {
        site <- paste0("https://data.snb.ch/api/cube/",
                       id, "/data/csv/", language)
    } else if (type == "dataset") {
        site <- paste0("https://data.snb.ch/api/warehouse/cube/",
                       gsub("@", ".", id),
                       "/data/csv/", language)
    } else {
        stop("either table of dataset must be specified")
    }

    if (!is.null(dest.dir))
        message("currently not supported")

    site <- paste0("https://data.snb.ch/api/cube/", table, "/lastUpdate")
    con <- url(site)
    ans <- try(readLines(con, warn = FALSE), silent = TRUE)
    try(close(con), silent = TRUE)
    em <- geterrmessage()
    if (requireNamespace("jsonlite"))
        jsonlite::fromJSON(ans)
    else
        ans
}

fetch_table_info <- function(table = "rendoblid",
                             dest.dir = NULL,
                             return.class = NULL,
                             verbose = TRUE,
                             language = "en",
                             method, ...) {
    info <- fetch_info(id = id, type = type,
                       language = language,
                       dest.dir = dest.dir,
                       name.sep = name.sep,
                       method = method)

    site <- paste0("https://data.snb.ch/api/cube/",
                   table, "/dimensions/",
                   language)
    if (!is.null(dest.dir)) {
        filename <- paste0(format(Sys.Date(), "%Y%m%d"),
                           "__", table,
                           "__info.csv")
                           "__", id,
                           "__", language,
                           ".csv")
        filename <- file.path(dest.dir, filename)
        if (!file.exists(filename)) {
            if (verbose)


@@ 55,6 54,7 @@ fetch_table_info <- function(table = "rendoblid",
        close(con)
        em <- geterrmessage()
    }

    if (inherits(dats, "try-error")) {
        if (verbose) {
            message("failed")


@@ 66,56 66,121 @@ fetch_table_info <- function(table = "rendoblid",
            message("done")
    }

    empty <- grep("^ *$", dats)
    header <- dats[1:empty]
    dats <- read.table(text = dats,
                       sep = ";",
                       header = TRUE,
                       stringsAsFactors = FALSE,
                       as.is = TRUE, skip = empty, ...)

    if (do.timeseries) {
        id <- dats[[2]][seq(1, min(which(duplicated(dats[[2]]))) - 1)]
        ans <- vector("list", length(id))
        names(ans) <- as.character(id)
        for (i in as.character(id)) {
            ans[[i]] <- dats[dats[, 2] == i, -2]
        }
        result <- do.call(cbind, lapply(ans, `[[`, 2))
        row.names(result) <- ans[[1]][[1]]

        if (na.drop) {
            drop <- apply(result, 1, function(x) all(!is.finite(x)))
            result <- result[!drop,, drop = FALSE]
        }

    } else {
        result <- dats
    }

    if (!is.null(return.class)) {
        stop("not yet supported")
        if (return.class == "zoo")
            if (requireNamespace("zoo"))
                stop("not yet implemented")
            else
                stop("package ", sQuote("zoo"), " not available")

        else if (return.class == "data.frame")
            result <- ans

        else if (return.class == "list")
            result <- NA

    }

    attr(result, "info") <- unlist(info)
    result
}



fetch_last_update <- function(id,
                              type = "table",
                              dataset,
                              dest.dir = NULL,
                              verbose = TRUE,
                              language = "en", ...) {
    if (!is.null(dest.dir))
        message("currently not supported")

    site <- if (type == "table") {
                paste0("https://data.snb.ch/api/cube/",
                       id, "/lastUpdate")
            } else if (type == "dataset") {
                paste0("https://data.snb.ch/api/warehouse/cube/",
                       gsub("@", ".", id), "/lastUpdate")
            }

    con <- url(site)
    ans <- try(readLines(con, warn = FALSE), silent = TRUE)
    try(close(con), silent = TRUE)
    em <- geterrmessage()
    if (requireNamespace("jsonlite"))
        jsonlite::fromJSON(dats, FALSE)
        jsonlite::fromJSON(ans)
    else
        dats
        ans
}

fetch_table <- function(table = "rendoblid",
                        dest.dir = NULL,
                        return.class = NULL,
                        verbose = TRUE,
                        method,
                        language = "en",
                        name.sep = " :: ",
                        na.drop = TRUE, ...) {


fetch_info <- function(id,
                       type = "table",
                       dest.dir = NULL,
                       verbose = TRUE,
                       language = "en",
                       name.sep = " :: ",
                       method, ...) {

    .do_item <- function(item, path = "", name.sep = " :: ") {
        if (length(item) == 2 &&
            identical(names(item), c("id", "name"))) {
        id.info[[item$id]] <<- paste0(path, name.sep, item$name)
    } else {
        if (!is.null(item$name))
            path <- paste0(path,
                           if (path != "") name.sep,
                           item$name)
        for (i in item) {
            if (is.list(i))
                .do_item(i, path, name.sep)
            id.info[[item$id]] <<- paste0(path, name.sep, item$name)
        } else {
            if (!is.null(item$name))
                path <- paste0(path,
                               if (path != "") name.sep,
                               item$name)
            for (i in item) {
                if (is.list(i))
                    .do_item(i, path, name.sep)
            }
        }
    }
}

    info <- fetch_table_info(table,
                             language = language,
                             dest.dir = dest.dir,
                             method = method)
    items <- info$dimensions ## overview
    id.info <- list()
    if (requireNamespace("jsonlite"))
        .do_item(items, path = "", name.sep = name.sep)
    else
        id.info <- info

    site <- paste0("https://data.snb.ch/api/cube/",
                   table, "/data/csv/", language)
    if (type == "table") {
        site <- paste0("https://data.snb.ch/api/cube/",
                       id, "/dimensions/", language)
    } else if (type == "dataset") {
        site <- paste0("https://data.snb.ch/api/warehouse/cube/",
                       gsub("@", ".", id), "/dimensions/", language)
    }

    if (!is.null(dest.dir)) {
        filename <- paste0(format(Sys.Date(), "%Y%m%d"),
                           "__", table,
                           ".csv")
                           "__", id,
                           "__", language,
                           "__info.csv")
        filename <- file.path(dest.dir, filename)
        if (!file.exists(filename)) {
            if (verbose)


@@ 127,7 192,7 @@ fetch_table <- function(table = "rendoblid",
            if (verbose)
                message("Using cache ... ", appendLF = FALSE)

        dats <- try(readLines(filename), silent = TRUE)
        dats <- try(readLines(filename, warn = FALSE), silent = TRUE)
        em <- geterrmessage()

    } else {


@@ 135,11 200,10 @@ fetch_table <- function(table = "rendoblid",
            message("Downloading data from SNB ... ", appendLF = FALSE)

        con <- url(site)
        dats <- try(readLines(con), silent = TRUE)
        dats <- try(readLines(con, warn = FALSE), silent = TRUE)
        close(con)
        em <- geterrmessage()
    }

    if (inherits(dats, "try-error")) {
        if (verbose) {
            message("failed")


@@ 151,43 215,335 @@ fetch_table <- function(table = "rendoblid",
            message("done")
    }

    empty <- grep("^ *$", dats)
    header <- dats[1:empty]
    dats <- read.table(text = dats,
                       sep = ";",
                       header = TRUE,
                       stringsAsFactors = FALSE,
                       as.is = TRUE, skip = empty, ...)
    if (requireNamespace("jsonlite")) {
        info <- jsonlite::fromJSON(dats, FALSE)
        items <- info$dimensions ## overview

    id <- dats[[2]][seq(1, min(which(duplicated(dats[[2]]))) - 1)]
    ans <- vector("list", length(id))
    names(ans) <- as.character(id)
    for (i in as.character(id)) {
        ans[[i]] <- dats[dats[, 2] == i, -2]
        id.info <- list()
        .do_item(items, path = "", name.sep = name.sep)
        dats <- id.info
    }
    result <- do.call(cbind, lapply(ans, `[[`, 2))
    row.names(result) <- ans[[1]][[1]]

    if (na.drop) {
        drop <- apply(result, 1, function(x) all(!is.finite(x)))
        result <- result[!drop,, drop = FALSE]
    }
    if (!is.null(return.class)) {
        stop("not yet supported")
        if (return.class == "zoo")
            if (requireNamespace("zoo"))
                stop("not yet implemented")
            else
                stop("package ", sQuote("zoo"), " not available")
    dats
}

        else if (return.class == "data.frame")
            result <- ans

        else if (return.class == "list")
            result <- NA

    }
## fetch_table_date <- function(table = "rendoblid",
##                              dest.dir = NULL,
##                              verbose = TRUE,
##                              language = "en", ...) {

    attr(result, "info") <- unlist(id.info)
    result
}
##     if (!is.null(dest.dir))
##         message("currently not supported")

##     site <- paste0("https://data.snb.ch/api/cube/", table, "/lastUpdate")
##     con <- url(site)
##     ans <- try(readLines(con, warn = FALSE), silent = TRUE)
##     try(close(con), silent = TRUE)
##     em <- geterrmessage()
##     if (requireNamespace("jsonlite"))
##         jsonlite::fromJSON(ans)
##     else
##         ans
## }



## fetch_table_info <- function(table = "rendoblid",
##                              dest.dir = NULL,
##                              return.class = NULL,
##                              verbose = TRUE,
##                              language = "en",
##                              method, ...) {

##     site <- paste0("https://data.snb.ch/api/cube/",
##                    table, "/dimensions/",
##                    language)
##     if (!is.null(dest.dir)) {
##         filename <- paste0(format(Sys.Date(), "%Y%m%d"),
##                            "__", table,
##                            "__", language,
##                            "__info.csv")
##         filename <- file.path(dest.dir, filename)
##         if (!file.exists(filename)) {
##             if (verbose)
##                 message("Downloading data from SNB ... ", appendLF = FALSE)
##             download.file(url = site, destfile = filename,
##                           method = method,
##                           quiet = TRUE)
##         } else
##             if (verbose)
##                 message("Using cache ... ", appendLF = FALSE)

##         dats <- try(readLines(filename, warn = FALSE), silent = TRUE)
##         em <- geterrmessage()

##     } else {
##         if (verbose)
##             message("Downloading data from SNB ... ", appendLF = FALSE)

##         con <- url(site)
##         dats <- try(readLines(con), silent = TRUE)
##         close(con)
##         em <- geterrmessage()
##     }

##     if (inherits(dats, "try-error")) {
##         if (verbose) {
##             message("failed")
##             message(em)
##         }
##         return(invisible(NULL))
##     } else {
##         if (verbose)
##             message("done")
##     }

##     if (requireNamespace("jsonlite"))
##         jsonlite::fromJSON(dats, FALSE)
##     else
##         dats
## }

## fetch_table <- function(table = "rendoblid",
##                         dest.dir = NULL,
##                         return.class = NULL,
##                         verbose = TRUE,
##                         method,
##                         language = "en",
##                         name.sep = " :: ",
##                         na.drop = TRUE, ...) {

##     .do_item <- function(item, path = "", name.sep = " :: ") {
##         if (length(item) == 2 &&
##             identical(names(item), c("id", "name"))) {
##             id.info[[item$id]] <<- paste0(path, name.sep, item$name)
##         } else {
##             if (!is.null(item$name))
##                 path <- paste0(path,
##                                if (path != "") name.sep,
##                                item$name)
##             for (i in item) {
##                 if (is.list(i))
##                     .do_item(i, path, name.sep)
##             }
##         }
##     }

##     info <- fetch_table_info(table,
##                              language = language,
##                              dest.dir = dest.dir,
##                              method = method)
##     items <- info$dimensions ## overview
##     id.info <- list()
##     if (requireNamespace("jsonlite"))
##         .do_item(items, path = "", name.sep = name.sep)
##     else
##         id.info <- info

##     site <- paste0("https://data.snb.ch/api/cube/",
##                    table, "/data/csv/", language)

##     if (!is.null(dest.dir)) {
##         filename <- paste0(format(Sys.Date(), "%Y%m%d"),
##                            "__", table,
##                            ".csv")
##         filename <- file.path(dest.dir, filename)
##         if (!file.exists(filename)) {
##             if (verbose)
##                 message("Downloading data from SNB ... ", appendLF = FALSE)
##             download.file(url = site, destfile = filename,
##                           method = method,
##                           quiet = TRUE)
##         } else
##             if (verbose)
##                 message("Using cache ... ", appendLF = FALSE)

##         dats <- try(readLines(filename), silent = TRUE)
##         em <- geterrmessage()

##     } else {
##         if (verbose)
##             message("Downloading data from SNB ... ", appendLF = FALSE)

##         con <- url(site)
##         dats <- try(readLines(con), silent = TRUE)
##         close(con)
##         em <- geterrmessage()
##     }

##     if (inherits(dats, "try-error")) {
##         if (verbose) {
##             message("failed")
##             message(em)
##         }
##         return(invisible(NULL))
##     } else {
##         if (verbose)
##             message("done")
##     }

##     empty <- grep("^ *$", dats)
##     header <- dats[1:empty]
##     dats <- read.table(text = dats,
##                        sep = ";",
##                        header = TRUE,
##                        stringsAsFactors = FALSE,
##                        as.is = TRUE, skip = empty, ...)

##     id <- dats[[2]][seq(1, min(which(duplicated(dats[[2]]))) - 1)]
##     ans <- vector("list", length(id))
##     names(ans) <- as.character(id)
##     for (i in as.character(id)) {
##         ans[[i]] <- dats[dats[, 2] == i, -2]
##     }
##     result <- do.call(cbind, lapply(ans, `[[`, 2))
##     row.names(result) <- ans[[1]][[1]]

##     if (na.drop) {
##         drop <- apply(result, 1, function(x) all(!is.finite(x)))
##         result <- result[!drop,, drop = FALSE]
##     }
##     if (!is.null(return.class)) {
##         stop("not yet supported")
##         if (return.class == "zoo")
##             if (requireNamespace("zoo"))
##                 stop("not yet implemented")
##             else
##                 stop("package ", sQuote("zoo"), " not available")

##         else if (return.class == "data.frame")
##             result <- ans

##         else if (return.class == "list")
##             result <- NA

##     }

##     attr(result, "info") <- unlist(id.info)
##     result
## }

## fetch_dataset <- function(dataset = "BSTA@SNB.JAHR_U.BIL.AKT.TOT",
##                           dest.dir = NULL,
##                           return.class = NULL,
##                           verbose = TRUE,
##                           method,
##                           language = "en",
##                           name.sep = " :: ",
##                           na.drop = TRUE, ...) {

##     .do_item <- function(item, path = "", name.sep = " :: ") {
##         if (length(item) == 2 &&
##             identical(names(item), c("id", "name"))) {
##             id.info[[item$id]] <<- paste0(path, name.sep, item$name)
##         } else {
##             if (!is.null(item$name))
##                 path <- paste0(path,
##                                if (path != "") name.sep,
##                                item$name)
##             for (i in item) {
##                 if (is.list(i))
##                     .do_item(i, path, name.sep)
##             }
##         }
##     }

##     info <- fetch_info(dataset = dataset,
##                        language = language,
##                        dest.dir = dest.dir,
##                        method = method)
##     items <- info$dimensions ## overview
##     id.info <- list()
##     if (requireNamespace("jsonlite"))
##         .do_item(items, path = "", name.sep = name.sep)
##     else
##         id.info <- info

##     site <- paste0("https://data.snb.ch/api/warehouse/cube/",
##                    gsub("@", ".", dataset),
##                    "/data/csv/", language)

##     if (!is.null(dest.dir)) {
##         filename <- paste0(format(Sys.Date(), "%Y%m%d"),
##                            "__", dataset,
##                            ".csv")
##         filename <- file.path(dest.dir, filename)
##         if (!file.exists(filename)) {
##             if (verbose)
##                 message("Downloading data from SNB ... ", appendLF = FALSE)
##             download.file(url = site, destfile = filename,
##                           method = method,
##                           quiet = TRUE)
##         } else
##             if (verbose)
##                 message("Using cache ... ", appendLF = FALSE)

##         dats <- try(readLines(filename), silent = TRUE)
##         em <- geterrmessage()

##     } else {
##         if (verbose)
##             message("Downloading data from SNB ... ", appendLF = FALSE)

##         con <- url(site)
##         dats <- try(readLines(con), silent = TRUE)
##         close(con)
##         em <- geterrmessage()
##     }

##     if (inherits(dats, "try-error")) {
##         if (verbose) {
##             message("failed")
##             message(em)
##         }
##         return(invisible(NULL))
##     } else {
##         if (verbose)
##             message("done")
##     }

##     empty <- grep("^ *$", dats)
##     header <- dats[1:(empty - 1)]
##     dats <- read.table(text = dats,
##                        sep = ";",
##                        header = TRUE,
##                        stringsAsFactors = FALSE,
##                        as.is = TRUE, skip = empty, ...)

##     ## id <- dats[[2]][seq(1, min(which(duplicated(dats[[2]]))) - 1)]
##     ## ans <- vector("list", length(id))
##     ## names(ans) <- as.character(id)
##     ## for (i in as.character(id)) {
##     ##     ans[[i]] <- dats[dats[, 2] == i, -2]
##     ## }
##     ## result <- do.call(cbind, lapply(ans, `[[`, 2))
##     ## row.names(result) <- ans[[1]][[1]]
##     result <- dats

##     if (na.drop) {
##         warning("ignored")

##         ## drop <- apply(result, 1, function(x) all(!is.finite(x)))
##         ## result <- result[!drop,, drop = FALSE]
##     }
##     if (!is.null(return.class)) {
##         stop("not yet supported")
##         if (return.class == "zoo")
##             if (requireNamespace("zoo"))
##                 stop("not yet implemented")
##             else
##                 stop("package ", sQuote("zoo"), " not available")

##         else if (return.class == "data.frame")
##             result <- ans

##         else if (return.class == "list")
##             result <- NA

##     }

##     attr(result, "info") <- unlist(id.info)
##     result
## }

M man/SNBdata.Rd => man/SNBdata.Rd +35 -27
@@ 1,9 1,9 @@
\name{SNBdata}
\alias{SNBdata-package}
\alias{SNBdata}
\alias{fetch_table}
\alias{fetch_table_date}
\alias{fetch_table_info}
\alias{fetch_data}
\alias{fetch_last_update}
\alias{fetch_info}
\concept{SNB}
\title{
  Download Data from the Swiss National Bank (SNB)


@@ 15,62 15,70 @@

}
\usage{
fetch_table(table = "rendoblid", dest.dir = NULL,
            return.class = NULL, verbose = TRUE,
            method,
            language = "en",
            name.sep = " :: ",
            na.drop = TRUE, ...)
fetch_table_date(table = "rendoblid",
                 dest.dir = NULL,
                 verbose = TRUE,
                 language = "en", ...)
fetch_table_info(table = "rendoblid", dest.dir = NULL,
                 return.class = NULL, verbose = TRUE,
                 language = "en", method, ...)
fetch_data(id, type = "table",
           dest.dir = NULL,
           return.class = NULL, verbose = TRUE,
           method,
           language = "en",
           name.sep = " :: ",
           na.drop = TRUE, ...)
fetch_last_update(id,
                  dest.dir = NULL,
                  verbose = TRUE,
                  language = "en", ...)
fetch_info(id, dest.dir = NULL,
           return.class = NULL, verbose = TRUE,
           language = "en", method, ...)
}
\arguments{
  \item{dest.dir}{%
    file path: where to store the files?
  }
  \item{table}{%
    the name of the table
  \item{id}{%
    string: identifier of the table/dataset
  }
  \item{type}{%
    string: \sQuote{\code{table}} or \sQuote{\code{dataset}}
  }
  \item{method}{%
   see \code{\link{download.file}}
  }
  \item{verbose}{
  \item{verbose}{%
    logical: print messages, e.g. about download progress?
  }
  \item{return.class}{\code{NULL} or character:
      not yet implemented (but in future versions, 
      not yet implemented (but in future versions,
    \code{\link[zoo]{zoo}} will be supported)
  }
  \item{language}{
  \item{language}{%
    string: \code{en}, \code{fr}, or \code{de}
  }
  \item{\dots}{
  \item{\dots}{%
    passed on to \code{\link{read.table}}
  }
  \item{name.sep}{
  \item{name.sep}{%
    string used when pasting description hierarchies
  }
  \item{na.drop}{
  \item{na.drop}{%
    logical. If \code{TRUE}, rows with no finite values at all
    are dropped.
  }
}
\details{

  The \acronym{SNB} provides data as tables and datasets
  The \acronym{SNB} provides data as tables and datasets.

}
\value{
  typically a \code{\link{data.frame}}.


}
\references{
  \url{https://data.snb.ch/en/help#data_api}
  \url{https://data.snb.ch/en/}

  for \code{help} on the \acronym{API},
  see\url{https://data.snb.ch/en/help#data_api}

}
\author{
  Enrico Schumann