~hrbrmstr/metis-tidy

c8450b643a0b4d72b5be1b489b808e206bfb53ab — boB Rudis 11 months ago 725bfd6
another go at Travis
8 files changed, 87 insertions(+), 79 deletions(-)

M DESCRIPTION
M NAMESPACE
M R/metis-tidy-package.R
M R/sql_translate_env.R
M man/dbplyr-interface.Rd
A man/sql_translate_env.AthenaConnection.Rd
M tests/test-all.R
D tests/testthat/test-metis.tidy.R
M DESCRIPTION => DESCRIPTION +6 -3
@@ 14,13 14,16 @@ Description: Methods are provided to use the 'metis' JDBC/DBI interface via
SystemRequirements: JDK 1.8+
License: MIT + file LICENSE
Suggests:
    testthat,
    covr
    covr,
    testthat
Depends:
    R (>= 3.2.0),
    metis
Imports:
    DBI,
    RJDBC,
    rJava,
    metis.jars,
    metis,
    dplyr,
    dbplyr
RoxygenNote: 6.1.1

M NAMESPACE => NAMESPACE +1 -0
@@ 5,6 5,7 @@ S3method(sql_translate_env,AthenaConnection)
import(DBI)
import(dbplyr)
import(metis)
import(metis.jars)
importFrom(dplyr,db_data_type)
importFrom(dplyr,sql_translate_env)
importFrom(dplyr,tbl)

M R/metis-tidy-package.R => R/metis-tidy-package.R +1 -1
@@ 9,7 9,7 @@
#' @keywords internal
#' @docType package
#' @author Bob Rudis (bob@@rud.is)
#' @import metis DBI dbplyr
#' @import metis.jars metis DBI dbplyr
#' @importFrom dplyr tbl db_data_type sql_translate_env
#' @references [Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide](https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf)
NULL

M R/sql_translate_env.R => R/sql_translate_env.R +2 -2
@@ 27,7 27,7 @@ db_data_type.AthenaConnection <- function(con, fields, ...) {

#' Translate R tridyverse functional idioms to Athena
#'
#' @rdname dbplyr-interface
#' @param con AthenaConnection
#' @export
sql_translate_env.AthenaConnection <- function(con) {



@@ 107,4 107,4 @@ sql_translate_env.AthenaConnection <- function(con) {

  )

}
\ No newline at end of file
}

M man/dbplyr-interface.Rd => man/dbplyr-interface.Rd +0 -5
@@ 2,12 2,9 @@
% Please edit documentation in R/sql_translate_env.R
\name{db_data_type.AthenaConnection}
\alias{db_data_type.AthenaConnection}
\alias{sql_translate_env.AthenaConnection}
\title{Convert R data type to Athena}
\usage{
\method{db_data_type}{AthenaConnection}(con, fields, ...)

\method{sql_translate_env}{AthenaConnection}(con)
}
\arguments{
\item{con}{Athena connection}


@@ 18,6 15,4 @@
}
\description{
Convert R data type to Athena

Translate R tridyverse functional idioms to Athena
}

A man/sql_translate_env.AthenaConnection.Rd => man/sql_translate_env.AthenaConnection.Rd +14 -0
@@ 0,0 1,14 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/sql_translate_env.R
\name{sql_translate_env.AthenaConnection}
\alias{sql_translate_env.AthenaConnection}
\title{Translate R tridyverse functional idioms to Athena}
\usage{
\method{sql_translate_env}{AthenaConnection}(con)
}
\arguments{
\item{con}{AthenaConnection}
}
\description{
Translate R tridyverse functional idioms to Athena
}

M tests/test-all.R => tests/test-all.R +63 -6
@@ 1,7 1,64 @@
library(metis.jars)
library(metis)
library(dbplyr)
library(dplyr)
library(testthat)
options(tidyverse.quiet=TRUE)

test_check("metis.tidy")
library(dbplyr, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE)
library(dplyr, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE)
library(metis.jars, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE)
library(metis, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE)
library(metis.tidy, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE)

testthat::context("d[b]plyr ops work as expected")

Sys.setenv(
  AWS_S3_STAGING_DIR = "s3://aws-athena-query-results-569593279821-us-east-1"
)

message("Making driver")
drv <- metis::Athena()
testthat::expect_is(drv, "AthenaDriver")

testthat::skip_on_cran()

message("Establishing connection")
if (identical(Sys.getenv("TRAVIS"), "true")) {

  metis::dbConnect(
    drv = drv,
    Schema = "sampledb",
    S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1"
  ) -> con

} else {

  metis::dbConnect(
    drv = drv,
    Schema = "sampledb",
    AwsCredentialsProviderClass = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider",
    AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props"),
    S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1",
  ) -> con

}

testthat::expect_is(con, "AthenaConnection")

message("Sourcing table")
elb_logs <- tbl(con, "elb_logs")

testthat::expect_is(elb_logs, "tbl_AthenaConnection")

message("Filtering and transforming")
filter(elb_logs, grepl("20", elbresponsecode)) %>%
  mutate(
    tsday = as.Date(substring(timestamp, 1L, 10L)),
    host = url_extract_host(url),
    proto_version = regexp_extract(protocol, "([[:digit:]\\.]+)"),
  ) %>%
  select(tsday, host, receivedbytes, requestprocessingtime, proto_version) %>%
  head(1) %>%
  collect() -> out

testthat::expect_is(out$tsday, "Date")
testthat::expect_is(out$host, "character")
testthat::expect_is(out$receivedbytes, "integer64")
testthat::expect_is(out$requestprocessingtime, "numeric")
testthat::expect_is(out$proto_version, "character")

D tests/testthat/test-metis.tidy.R => tests/testthat/test-metis.tidy.R +0 -62
@@ 1,62 0,0 @@
context("d[b]plyr ops work as expected")

Sys.setenv(
  AWS_S3_STAGING_DIR = "s3://aws-athena-query-results-569593279821-us-east-1"
)

library(metis)
library(dbplyr)
library(dplyr)

drv <- metis::Athena()

skip_on_cran()

if (identical(Sys.getenv("TRAVIS"), "true")) {

  metis::dbConnect(
    drv = drv,
    Schema = "sampledb",
    S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1"
  ) -> con

} else {

  metis::dbConnect(
    drv = drv,
    Schema = "sampledb",
    AwsCredentialsProviderClass = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider",
    AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props"),
    S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1",
  ) -> con

}

metis::dbConnect(
  drv,
  Schema = "sampledb",
  AwsCredentialsProviderClass = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider",
  AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props")
) -> con

expect_is(con, "AthenaConnection")

elb_logs <- tbl(con, "elb_logs")

expect_is(elb_logs, "tbl_AthenaConnection")

filter(elb_logs, grepl("20", elbresponsecode)) %>%
  mutate(
    tsday = as.Date(substring(timestamp, 1L, 10L)),
    host = url_extract_host(url),
    proto_version = regexp_extract(protocol, "([[:digit:]\\.]+)"),
  ) %>%
  select(tsday, host, receivedbytes, requestprocessingtime, proto_version) %>%
  head(1) %>%
  collect() -> out

expect_is(out$tsday, "Date")
expect_is(out$host, "character")
expect_is(out$receivedbytes, "integer64")
expect_is(out$requestprocessingtime, "numeric")
expect_is(out$proto_version, "character")