M .Rbuildignore => .Rbuildignore +2 -0
@@ 1,3 1,5 @@
+^CODE_OF_CONDUCT\.md$
+^LICENSE\.md$
^.*\.Rproj$
^\.Rproj\.user$
^\.travis\.yml$
A CODE_OF_CONDUCT.md => CODE_OF_CONDUCT.md +25 -0
@@ 0,0 1,25 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who
+contribute through reporting issues, posting feature requests, updating documentation,
+submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for
+everyone, regardless of level of experience, gender, gender identity and expression,
+sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or
+imagery, derogatory comments or personal attacks, trolling, public or private harassment,
+insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments,
+commits, code, wiki edits, issues, and other contributions that are not aligned to this
+Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed
+from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by
+opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the Contributor Covenant
+(http://contributor-covenant.org), version 1.0.0, available at
+http://contributor-covenant.org/version/1/0/0/
M DESCRIPTION => DESCRIPTION +5 -4
@@ 13,13 13,14 @@ Description: The <urlscan.io> service provides an 'API' enabling analysis of
browser, <urlscan.io> will let you take a look at the individual resources
that are requested when a site is loaded. Tools are provided to search
public <urlscans.io> scan submissions/results and submit URLs for scanning.
-URL: https://github.com/hrbrmstr/urlscan
-BugReports: https://github.com/hrbrmstr/urlscan/issues
+URL: https://gitlab.com/hrbrmstr/urlscan
+BugReports: https://gitlab.com/hrbrmstr/urlscan/issues
Encoding: UTF-8
-License: AGPL
+License: MIT + file LICENSE
Suggests:
testthat,
- covr
+ covr,
+ pingr
Depends:
R (>= 3.2.0)
Imports:
A LICENSE => LICENSE +2 -0
@@ 0,0 1,2 @@
+YEAR: 2019
+COPYRIGHT HOLDER: Bob Rudis
A LICENSE.md => LICENSE.md +21 -0
@@ 0,0 1,21 @@
+# MIT License
+
+Copyright (c) 2019 Bob Rudis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
M NEWS.md => NEWS.md +2 -0
@@ 3,6 3,8 @@
* Changed user agent generation
* Updated for the new API including support for authentication
* Added object printers
+* MIT license
+* Added tests
0.1.0
* Initial release
M R/results.R => R/results.R +3 -0
@@ 14,6 14,9 @@
#' The list can also include `screenshot` if `include_shot` is `TRUE` and
#' a screenshot was available.
#' @export
+#' @examples
+#' x <- urlscan_search("domain:r-project.org")
+#' r <- urlscan_result(x$results[["_id"]][1])
urlscan_result <- function(scan_id, include_dom=FALSE, include_shot=FALSE) {
httr::GET(
M R/submit.R => R/submit.R +3 -0
@@ 21,6 21,9 @@
#' he scan, you can use it to retrieve the result after waiting for a short while.
#' @references <https://urlscan.io/about-api/#submission>
#' @export
+#' @examples \dontrun{
+#' urlscan_submit("https://r-project.org/")
+#' }
urlscan_submit <- function(url, public=TRUE, custom_agent=NULL, referer=NULL,
api_key = urlscan_api_key()) {
M R/urlscan.R => R/urlscan.R +2 -0
@@ 24,6 24,8 @@
#' @references <https://urlscan.io/about-api/#search>
#' @note Search can only find **public** scans, there is no way to search for private scans.
#' @export
+#' @examples
+#' x <- urlscan_search("r-project.org")
urlscan_search <- function(query, size=100, offset=0, sort=NULL) {
httr::GET(
D R/util.R => R/util.R +0 -12
@@ 1,12 0,0 @@
-#' #' Turn urlscan object into a data frame
-#' #'
-#' #' param x `urlscan` object
-#' #' param ... unused
-#' #' export
-#' as.data.frame.urlscan <- function(x, ...) {
-#'
-#' res <- x$results
-#' class(res) <- c("tbl_df", "tbl", "data.frame")
-#' res
-#'
-#' }>
\ No newline at end of file
M README.Rmd => README.Rmd +14 -1
@@ 3,6 3,9 @@ output: rmarkdown::github_document
editor_options:
chunk_output_type: console
---
+[](https://travis-ci.org/hrbrmstr/urlscan)
+[](https://codecov.io/gh/hrbrmstr/urlscan)
+[](https://cran.r-project.org/package=urlscan)
# urlscan
@@ 67,4 70,14 @@ ures
magick::image_write(ures$screenshot, "img/shot.png")
```
->
\ No newline at end of file
+
+
+## urlscan Metrics
+
+```{r cloc, echo=FALSE}
+cloc::cloc_pkg_md()
+```
+
+## Code of Conduct
+
+Please note that the 'urlscan' project is released with a [Contributor Code of Conduct](CODE_OF_CONDUCT.md). By contributing to this project, you agree to abide by its terms.<
\ No newline at end of file
M README.md => README.md +19 -0
@@ 1,4 1,10 @@
+[](https://travis-ci.org/hrbrmstr/urlscan)
+[](https://codecov.io/gh/hrbrmstr/urlscan)
+[](https://cran.r-project.org/package=urlscan)
+
# urlscan
Analyze Websites and Resources They Request
@@ 72,3 78,16 @@ magick::image_write(ures$screenshot, "img/shot.png")
```

+
+## urlscan Metrics
+
+| Lang | \# Files | (%) | LoC | (%) | Blank lines | (%) | \# Lines | (%) |
+| :--- | -------: | ---: | --: | ---: | ----------: | ---: | -------: | ---: |
+| R | 10 | 0.91 | 157 | 0.89 | 51 | 0.69 | 130 | 0.76 |
+| Rmd | 1 | 0.09 | 20 | 0.11 | 23 | 0.31 | 40 | 0.24 |
+
+## Code of Conduct
+
+Please note that the ‘urlscan’ project is released with a [Contributor
+Code of Conduct](CODE_OF_CONDUCT.md). By contributing to this project,
+you agree to abide by its terms.
M img/shot.png => img/shot.png +0 -0
M man/urlscan_result.Rd => man/urlscan_result.Rd +4 -0
@@ 27,3 27,7 @@ a screenshot was available.
\description{
Retrieve detailed results for a given scan ID
}
+\examples{
+x <- urlscan_search("domain:r-project.org")
+r <- urlscan_result(x$results[["_id"]][1])
+}
M man/urlscan_search.Rd => man/urlscan_search.Rd +3 -0
@@ 38,6 38,9 @@ Furthermore, you can concatenate search-terms with \code{AND}, \code{OR}, etc.
\note{
Search can only find \strong{public} scans, there is no way to search for private scans.
}
+\examples{
+x <- urlscan_search("r-project.org")
+}
\references{
\url{https://urlscan.io/about-api/#search}
}
M man/urlscan_submit.Rd => man/urlscan_submit.Rd +5 -0
@@ 34,6 34,11 @@ The API key should be in the environment variable \code{URLSCAN_API_KEY} and the
easiest way to do that is via the \code{~/.Renviron} file. You can also manually pass
it in as a parameter.
}
+\examples{
+\dontrun{
+urlscan_submit("https://r-project.org/")
+}
+}
\references{
\url{https://urlscan.io/about-api/#submission}
}
M tests/testthat/test-urlscan.R => tests/testthat/test-urlscan.R +17 -2
@@ 1,6 1,21 @@
-context("minimal package functionality")
+context("urlscan api interaction works")
test_that("we can do something", {
- #expect_that(some_function(), is_a("data.frame"))
+ skip_if(
+ inherits(
+ try(pingr::ping_port("google.com"), silent = TRUE),
+ "try-error"
+ ),
+ message = "Skipping due to internet connection not detected"
+ )
+
+ x <- urlscan_search("r-project")
+
+ expect_is(x, "urlscan")
+ expect_true(length(x$results$stats) > 0)
+
+ r <- urlscan_result(x$results[["_id"]][1])
+ expect_is(r, "urlscan_result")
+ expect_true(length(r$scan_result$task) > 0)
})