16 files changed, 99 insertions(+), 30 deletions(-)

M .Rbuildignore
M DESCRIPTION
A LICENSE
A LICENSE.md
M NAMESPACE
M NEWS.md
M R/apache-httpd.R
M R/chrome.R
M R/ios.R
M R/memcached.R
M R/mongodb.R
M R/nginx.R
M R/openssh.R
M R/php.R
M R/sendmail.R
M R/vershist-package.R

@@ 1,3 1,4 @@
+^LICENSE\.md$
 ^.*\.Rproj$
 ^\.Rproj\.user$
 ^\.travis\.yml$

@@ 1,8 1,8 @@
 Package: vershist
 Type: Package
 Title: Collect Version Histories For Vendor Products
-Version: 0.2.2
-Date: 2019-01-27
+Version: 0.3.0
+Date: 2019-02-08
 Authors@R: c(
     person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"), 
            comment = c(ORCID = "0000-0001-5670-2640"))


@@ 14,17 14,18 @@ URL: https://gitlab.com/hrbrmstr/vershist
 BugReports: https://gitlab.com/hrbrmstr/vershist/issues
 SystemRequirements: C++11
 Encoding: UTF-8
-License: AGPL
+License: MIT + file LICENSE
 Suggests:
     testthat,
-    covr
+    covr,
+    tibble (>= 2.0.1)
 Depends:
     R (>= 3.2.0)
 Imports:
     purrr,
     rvest,
     readr,
-    dplyr,
+    dplyr (>= 0.7.8),
     stringi,
     semver,
     lubridate,

@@ 0,0 1,2 @@
+YEAR: 2019
+COPYRIGHT HOLDER: Bob Rudis

@@ 0,0 1,21 @@
+# MIT License
+
+Copyright (c) 2019 Bob Rudis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

@@ 22,7 22,7 @@ import(semver)
 importFrom(Rcpp,sourceCpp)
 importFrom(curl,curl)
 importFrom(dplyr,arrange)
-importFrom(dplyr,as_data_frame)
+importFrom(dplyr,as_tibble)
 importFrom(dplyr,bind_cols)
 importFrom(dplyr,data_frame)
 importFrom(dplyr,distinct)


@@ 67,6 67,7 @@ importFrom(stringi,stri_replace_first_regex)
 importFrom(stringi,stri_replace_last_fixed)
 importFrom(stringi,stri_replace_last_regex)
 importFrom(stringi,stri_split_fixed)
+importFrom(stringi,stri_split_lines)
 importFrom(stringi,stri_sub)
 importFrom(stringi,stri_trans_tolower)
 importFrom(stringi,stri_trim_both)

@@ 1,3 1,8 @@
+0.3.0
+* Significant improvements to Apache and nginx version retrievers
+* Swapped out as_tibble for as_data_frame
+* MIT license
+
 0.2.2
 * Support for ISC BIND
 

@@ 10,20 10,32 @@
 #' @export
 apache_httpd_version_history <- function() {
 
-  readr::read_lines("https://archive.apache.org/dist/httpd/") %>%
-    purrr::keep(stri_detect_regex, 'apache_.*gz"') %>%
-    stri_replace_first_fixed("<img src=\"/icons/compressed.gif\" alt=\"[   ]\"> ", "") %>%
-    stri_match_first_regex('href="apache_([[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+)\\.tar\\.gz".*([[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2})') %>%
-    dplyr::as_data_frame() %>%
-    dplyr::select(-V1) %>%
-    dplyr::rename(vers = V2, rls_date = V3) %>%
-    dplyr::mutate(rls_date = as.Date(rls_date)) %>%
-    dplyr::mutate(rls_year = lubridate::year(rls_date)) %>%
+  ap <- readr::read_lines("https://archive.apache.org/dist/httpd/")
+
+  apd <- xml2::read_html(paste0(ap[grepl('"(httpd-|apache_)[[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+\\.tar', ap)], collapse="\n"))
+
+  rvest::html_text(apd) %>%
+    stri_split_lines() %>%
+    unlist() %>%
+    stri_match_first_regex("([[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2})") %>%
+    .[,2] -> rls_dates
+
+  rvest::html_nodes(apd, "a") %>%
+    rvest::html_attr("href") %>%
+    stri_match_first_regex("[-_]([[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+)\\.tar") %>%
+    .[,2] -> vers
+
+  dplyr::tibble(
+    vers = vers,
+    rls_date = rls_dates
+  ) %>%
+    dplyr::distinct(vers, .keep_all=TRUE) %>%
+    mutate(rls_date = as.Date(rls_date)) %>%
+    mutate(rls_year = lubridate::year(rls_date)) %>%
     dplyr::bind_cols(
       semver::parse_version(.$vers) %>%
-        dplyr::as_data_frame()
+        dplyr::as_tibble()
     ) %>%
     dplyr::arrange(major, minor, patch) %>%
     dplyr::mutate(vers = factor(vers, levels = vers))
-
 }

@@ 29,7 29,7 @@ google_chrome_version_history <- function() {
     dplyr::distinct(vers, .keep_all=TRUE) %>%
     dplyr::bind_cols(
       semver::parse_version(.$vers) %>%
-        dplyr::as_data_frame()
+        dplyr::as_tibble()
     ) %>%
     dplyr::arrange(major, minor, patch) %>%
     dplyr::mutate(vers = factor(vers, levels = vers))

@@ 54,7 54,7 @@ apple_ios_version_history <- function() {
     ) %>%
     dplyr::bind_cols(
       semver::parse_version(.$vers) %>%
-        dplyr::as_data_frame()
+        dplyr::as_tibble()
     ) %>%
     dplyr::arrange(major, minor, patch) %>%
     dplyr::mutate(vers = factor(vers, levels = vers))

@@ 39,14 39,14 @@ memcached_version_history <- function() {
     purrr::keep(stri_detect_fixed, "[[ReleaseNotes") %>%
     stri_replace_first_regex(" \\* \\[\\[.*]] ", "") %>%
     stri_split_fixed(" ", 2, simplify = TRUE) %>%
-    dplyr::as_data_frame() %>%
+    dplyr::as_tibble() %>%
     purrr::set_names(c("vers", "rls_date")) %>%
     dplyr::mutate(string = stri_trim_both(vers)) %>%
     dplyr::mutate(rls_date = fix_memcached_dates(rls_date)) %>%
     dplyr::mutate(rls_year = lubridate::year(rls_date)) %>%
     dplyr::bind_cols(
       semver::parse_version(.$vers) %>%
-        dplyr::as_data_frame()
+        dplyr::as_tibble()
     ) %>%
     dplyr::arrange(major, minor, patch) %>%
     dplyr::mutate(vers = factor(vers, levels = vers))

@@ 35,7 35,7 @@ mongodb_version_history <- function() {
     dplyr::mutate(year = lubridate::year(ts)) %>%
     dplyr::bind_cols(
       semver::parse_version(.$vers) %>%
-        dplyr::as_data_frame()
+        dplyr::as_tibble()
     ) %>%
     dplyr::arrange(major, minor, patch) %>%
     dplyr::mutate(vers = factor(vers, levels=vers)) %>%

@@ 15,17 15,43 @@ nginx_version_history <- function() {
 
   doc <- suppressWarnings(xml2::read_xml(nginx_changes_url))
 
-  dplyr::data_frame(
+  dplyr::tibble(
     vers = rvest::xml_nodes(doc, xpath="//changes") %>%
       xml2::xml_attr("ver"),
     ts = rvest::xml_nodes(doc, xpath="//changes") %>%
       xml2::xml_attr("date") %>%
       as.Date(),
     year = lubridate::year(ts)
+  ) -> c1
+
+  c(
+    readr::read_lines("https://nginx.org/en/CHANGES-1.0"),
+    readr::read_lines("https://nginx.org/en/CHANGES-1.2"),
+    readr::read_lines("https://nginx.org/en/CHANGES-1.4"),
+    readr::read_lines("https://nginx.org/en/CHANGES-1.6"),
+    readr::read_lines("https://nginx.org/en/CHANGES-1.8"),
+    readr::read_lines("https://nginx.org/en/CHANGES-1.10"),
+    readr::read_lines("https://nginx.org/en/CHANGES-1.12"),
+    readr::read_lines("https://nginx.org/en/CHANGES-1.14")
+  ) -> nl
+
+  read.csv(
+    stringsAsFactors = FALSE,
+    text = paste0(
+      c("v,d", nl[grepl("^Changes", nl)] %>%
+          gsub("Changes with nginx ", "", .) %>%
+          gsub("[[:space:]]{3,}", ",", .)), collapse="\n")
   ) %>%
+    dplyr::as_tibble() %>%
+    dplyr::mutate(d = lubridate::dmy(d)) %>%
+    dplyr::select(vers=1, ts=2) %>%
+    dplyr::mutate(year = lubridate::year(ts)) -> c2
+
+  dplyr::bind_rows(c1, c2) %>%
+    dplyr::distinct() %>%
     dplyr::bind_cols(
       semver::parse_version(.$vers) %>%
-        dplyr::as_data_frame()
+        dplyr::as_tibble()
     ) %>%
     dplyr::arrange(major, minor, patch) %>%
     dplyr::mutate(vers = factor(vers, levels=vers)) %>%

@@ 35,7 35,7 @@ openssh_version_history <- function() {
     ) %>%
     dplyr::bind_cols(
       semver::parse_version(.$vers) %>%
-        dplyr::as_data_frame()
+        dplyr::as_tibble()
     ) %>%
     dplyr::arrange(major, minor, patch) %>%
     dplyr::mutate(vers = factor(vers, levels = vers))

@@ 27,7 27,7 @@ php_version_history <- function() {
   ) %>%
     dplyr::bind_cols(
       semver::parse_version(.$vers) %>%
-        dplyr::as_data_frame()
+        dplyr::as_tibble()
     ) %>%
     dplyr::arrange(major, minor, patch) %>%
     dplyr::mutate(vers = factor(vers, levels=vers)) %>%

@@ 15,7 15,7 @@ sendmail_version_history <- function() {
   close(con)
 
   stri_match_first_regex(res, "([[:alpha:]]{3} [[:digit:]]{2}  [[:digit:]]{4}) (.*)") %>%
-    dplyr::as_data_frame() %>%
+    dplyr::as_tibble() %>%
     dplyr::select(-V1) %>%
     dplyr::rename(vers = V3, ts = V2) %>%
     dplyr::select(vers, ts) %>%


@@ 28,7 28,7 @@ sendmail_version_history <- function() {
     dplyr::rename(rls_date = ts, rls_year = year) %>%
     dplyr::bind_cols(
       semver::parse_version(.$vers) %>%
-        dplyr::as_data_frame()
+        dplyr::as_tibble()
     ) %>%
     dplyr::arrange(major, minor, patch) %>%
     dplyr::mutate(vers = factor(vers, levels=vers))

@@ 9,11 9,11 @@
 #' @author Bob Rudis (bob@@rud.is)
 #' @import semver
 #' @importFrom purrr keep discard map map_df %>% safely set_names
-#' @importFrom dplyr mutate rename select as_data_frame left_join bind_cols arrange
+#' @importFrom dplyr mutate rename select as_tibble left_join bind_cols arrange
 #' @importFrom dplyr rename progress_estimated mutate_at distinct data_frame
 #' @importFrom stringi stri_match_first_regex stri_detect_fixed stri_detect_regex
 #' @importFrom stringi stri_replace_all_regex stri_replace_first_fixed stri_trans_tolower
-#' @importFrom stringi stri_extract_first_regex stri_sub stri_replace_first_regex
+#' @importFrom stringi stri_extract_first_regex stri_sub stri_replace_first_regex stri_split_lines
 #' @importFrom stringi stri_replace_all_fixed stri_split_fixed stri_count_fixed stri_trim_both
 #' @importFrom stringi stri_extract_all_regex stri_replace_last_regex stri_replace_last_fixed
 #' @importFrom lubridate year mdy mdy_hms parse_date_time