~ninjin/julia-nix

2f47dae3623fb569e045902bd9caba48ca816f6c — Elliot Saba 2 years ago 5fb27b2
Move `SHA` to be an external stdlib (#41370)

For the past few years, we have had `SHA` as a vendored copy, however
improvements have been made both to the vendored copy here in base as
well as to the registered copy living within `JuliaCrypto`.  This
eliminates the vendored copy and sets `SHA` up as an external stdlib
just like `Pkg`, `Tar`, and all of their friends, as well as merges the
improvements made to both into the external repository.
19 files changed, 9 insertions(+), 1498 deletions(-)

A deps/checksums/SHA-7ac490dfa78449ae81354cff9471a317f211a8f6.tar.gz/md5
A deps/checksums/SHA-7ac490dfa78449ae81354cff9471a317f211a8f6.tar.gz/sha512
M stdlib/.gitignore
M stdlib/Makefile
A stdlib/SHA.version
D stdlib/SHA/LICENSE.md
D stdlib/SHA/Project.toml
D stdlib/SHA/docs/src/index.md
D stdlib/SHA/src/SHA.jl
D stdlib/SHA/src/base_functions.jl
D stdlib/SHA/src/common.jl
D stdlib/SHA/src/constants.jl
D stdlib/SHA/src/hmac.jl
D stdlib/SHA/src/sha1.jl
D stdlib/SHA/src/sha2.jl
D stdlib/SHA/src/sha3.jl
D stdlib/SHA/src/types.jl
D stdlib/SHA/test/perf.jl
D stdlib/SHA/test/runtests.jl
A deps/checksums/SHA-7ac490dfa78449ae81354cff9471a317f211a8f6.tar.gz/md5 => deps/checksums/SHA-7ac490dfa78449ae81354cff9471a317f211a8f6.tar.gz/md5 +1 -0
@@ 0,0 1,1 @@
db2a92db568240cfe668744e91cdc649

A deps/checksums/SHA-7ac490dfa78449ae81354cff9471a317f211a8f6.tar.gz/sha512 => deps/checksums/SHA-7ac490dfa78449ae81354cff9471a317f211a8f6.tar.gz/sha512 +1 -0
@@ 0,0 1,1 @@
014f1dd31528b5c0618f5d2373f855d56e3b30f7b09cb1d680137e6254be48e9384a1950d066d59e599a86743922ab2c24d812b2faf294c62d6cb80bbe9fd706

M stdlib/.gitignore => stdlib/.gitignore +2 -0
@@ 15,5 15,7 @@
/NetworkOptions
/SuiteSparse-*
/SuiteSparse
/SHA-*
/SHA
/*_jll/StdlibArtifacts.toml
/*/Manifest.toml

M stdlib/Makefile => stdlib/Makefile +3 -1
@@ 44,7 44,7 @@ STDLIBS = Artifacts Base64 CRC32c Dates DelimitedFiles Distributed FileWatching 
          SharedArrays Sockets SparseArrays SuiteSparse Test TOML Unicode UUIDs \
          $(JLL_NAMES)

STDLIBS_EXT = Pkg Statistics LibCURL Downloads ArgTools Tar NetworkOptions SuiteSparse
STDLIBS_EXT = Pkg Statistics LibCURL Downloads ArgTools Tar NetworkOptions SuiteSparse SHA
PKG_GIT_URL := git://github.com/JuliaLang/Pkg.jl.git
PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1
STATISTICS_GIT_URL := git://github.com/JuliaLang/Statistics.jl.git


@@ 61,6 61,8 @@ NETWORKOPTIONS_GIT_URL := git://github.com/JuliaLang/NetworkOptions.jl.git
NETWORKOPTIONS_TAR_URL = https://api.github.com/repos/JuliaLang/NetworkOptions.jl/tarball/$1
SUITESPARSE_GIT_URL := git://github.com/JuliaLang/SuiteSparse.jl.git
SUITESPARSE_TAR_URL = https://api.github.com/repos/JuliaLang/SuiteSparse.jl/tarball/$1
SHA_GIT_URL := git://github.com/JuliaCrypto/SHA.jl.git
SHA_TAR_URL = https://api.github.com/repos/JuliaCrypto/SHA.jl/tarball/$1

$(foreach module, $(STDLIBS_EXT), $(eval $(call stdlib-external,$(module),$(shell echo $(module) | tr a-z A-Z))))


A stdlib/SHA.version => stdlib/SHA.version +2 -0
@@ 0,0 1,2 @@
SHA_BRANCH = master
SHA_SHA1 = 7ac490dfa78449ae81354cff9471a317f211a8f6

D stdlib/SHA/LICENSE.md => stdlib/SHA/LICENSE.md +0 -58
@@ 1,58 0,0 @@
The SHA.jl package is licensed under the MIT "Expat" License:

> Copyright (c) 2014: Elliot Saba.
>
> Permission is hereby granted, free of charge, to any person obtaining
> a copy of this software and associated documentation files (the
> "Software"), to deal in the Software without restriction, including
> without limitation the rights to use, copy, modify, merge, publish,
> distribute, sublicense, and/or sell copies of the Software, and to
> permit persons to whom the Software is furnished to do so, subject to
> the following conditions:
>
> The above copyright notice and this permission notice shall be
> included in all copies or substantial portions of the Software.
>
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

This package was inspired by the SHA2 [source code from Minix](https://github.com/minix3/minix/blob/b6cbf7203b080219de306404f8022a65b7884f33/common/lib/libc/hash/sha2/sha2.c), itself released under the BSD license:

> sha2.c
>
> Version 1.0.0beta1
>
> Written by Aaron D. Gifford <me@aarongifford.com>
>
> Copyright 2000 Aaron D. Gifford.  All rights reserved.
>
> Redistribution and use in source and binary forms, with or without
> modification, are permitted provided that the following conditions
> are met:
>
> 1. Redistributions of source code must retain the above copyright
>    notice, this list of conditions and the following disclaimer.
>
> 2. Redistributions in binary form must reproduce the above copyright
>    notice, this list of conditions and the following disclaimer in the
>    documentation and/or other materials provided with the distribution.
>
> 3. Neither the name of the copyright holder nor the names of contributors
>    may be used to endorse or promote products derived from this software
>    without specific prior written permission.
>
> THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) AND CONTRIBUTOR(S) ``AS IS'' AND
> ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
> IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
> ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR(S) OR CONTRIBUTOR(S) BE LIABLE
> FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
> DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
> OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
> HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
> LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
> OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF

D stdlib/SHA/Project.toml => stdlib/SHA/Project.toml +0 -8
@@ 1,8 0,0 @@
name = "SHA"
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[extras]
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[targets]
test = ["Test"]

D stdlib/SHA/docs/src/index.md => stdlib/SHA/docs/src/index.md +0 -75
@@ 1,75 0,0 @@
# SHA


Usage is very straightforward:
```julia
julia> using SHA

julia> bytes2hex(sha256("test"))
"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
```

Each exported function (at the time of this writing, SHA-1, SHA-2 224, 256, 384 and 512, and SHA-3 224, 256, 384 and 512 functions are implemented) takes in either an `AbstractVector{UInt8}`, an `AbstractString` or an `IO` object.  This makes it trivial to checksum a file:

```julia
shell> cat /tmp/test.txt
test
julia> using SHA

julia> open("/tmp/test.txt") do f
           sha2_256(f)
       end
32-element Array{UInt8,1}:
 0x9f
 0x86
 0xd0
 0x81
 0x88
 0x4c
 0x7d
 0x65
 0x5d
 0x6c
 0x15
 0xb0
 0xf0
 0x0a
 0x08
```

Due to the colloquial usage of `sha256` to refer to `sha2_256`, convenience functions are provided, mapping `shaxxx()` function calls to `sha2_xxx()`.  For SHA-3, no such colloquialisms exist and the user must use the full `sha3_xxx()` names.

`shaxxx()` takes `AbstractString` and array-like objects (`NTuple` and `Array`) with elements of type `UInt8`.

To create a hash from multiple items the `SHAX_XXX_CTX()` types can be used to create a stateful hash object that
is updated with `update!` and finalized with `digest!`

```julia
julia> ctx = SHA2_256_CTX()
SHA2 256-bit hash state

julia> update!(ctx, b"some data")
0x0000000000000009

julia> update!(ctx, b"some more data")
0x0000000000000017

julia> digest!(ctx)
32-element Vector{UInt8}:
 0xbe
 0xcf
 0x23
 0xda
 0xaf
 0x02
 0x25
 0x52
 0x19
 0xa0
 0x8b
 0xc5
```

Note that, at the time of this writing, the SHA3 code is not optimized, and as such is roughly an order of magnitude slower than SHA2.

D stdlib/SHA/src/SHA.jl => stdlib/SHA/src/SHA.jl +0 -137
@@ 1,137 0,0 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

"""
    SHA

The SHA module provides hashing functionality for SHA1, SHA2 and SHA3 algorithms.

They are implemented as both pure functions for hashing single pieces of data,
or a stateful context which can be updated with the `update!` function and
finalized with `digest!`.

```julia-repl
julia> sha1(b"some data")
20-element Vector{UInt8}:
 0xba
 0xf3
 0xe3
 0x56


julia> ctx = SHA1_CTX()
SHA1 hash state

julia> update!(ctx, b"some data")
0x0000000000000009

julia> digest!(ctx)
20-element Vector{UInt8}:
 0xba
 0xf3
 0xe3
 0x56
"""
module SHA

# Export convenience functions, context types, update!() and digest!() functions
export sha1, SHA1_CTX, update!, digest!
export sha224, sha256, sha384, sha512
export sha2_224, sha2_256, sha2_384, sha2_512
export sha3_224, sha3_256, sha3_384, sha3_512
export SHA224_CTX, SHA256_CTX, SHA384_CTX, SHA512_CTX
export SHA2_224_CTX, SHA2_256_CTX, SHA2_384_CTX, SHA2_512_CTX
export SHA3_224_CTX, SHA3_256_CTX, SHA3_384_CTX, SHA3_512_CTX
export HMAC_CTX, hmac_sha1
export hmac_sha224, hmac_sha256, hmac_sha384, hmac_sha512
export hmac_sha2_224, hmac_sha2_256, hmac_sha2_384, hmac_sha2_512
export hmac_sha3_224, hmac_sha3_256, hmac_sha3_384, hmac_sha3_512

# data to be hashed:
const AbstractBytes = Union{AbstractVector{UInt8},NTuple{N,UInt8} where N}

include("constants.jl")
include("types.jl")
include("base_functions.jl")
include("sha1.jl")
include("sha2.jl")
include("sha3.jl")
include("common.jl")
include("hmac.jl")

# Create data types and convenience functions for each hash implemented
for (f, ctx) in [(:sha1, :SHA1_CTX),
                 (:sha224, :SHA224_CTX),
                 (:sha256, :SHA256_CTX),
                 (:sha384, :SHA384_CTX),
                 (:sha512, :SHA512_CTX),
                 (:sha2_224, :SHA2_224_CTX),
                 (:sha2_256, :SHA2_256_CTX),
                 (:sha2_384, :SHA2_384_CTX),
                 (:sha2_512, :SHA2_512_CTX),
                 (:sha3_224, :SHA3_224_CTX),
                 (:sha3_256, :SHA3_256_CTX),
                 (:sha3_384, :SHA3_384_CTX),
                 (:sha3_512, :SHA3_512_CTX),]
    g = Symbol(:hmac_, f)

    @eval begin
        # Our basic function is to process arrays of bytes
        """
            $($f)(data)

        Hash data using the $($f) algorithm and return the resulting digest.
        See also [`$($ctx)`](@ref).
        """
        function $f(data::AbstractBytes)
            ctx = $ctx()
            update!(ctx, data)
            return digest!(ctx)

        """
            $($g)(key, data)

        Hash data using the $($f) algorithm using the passed key
        See also [`HMAC_CTX`](@ref).
        """
        end
        function $g(key::Vector{UInt8}, data::AbstractBytes)
            ctx = HMAC_CTX($ctx(), key)
            update!(ctx, data)
            return digest!(ctx)
        end

        # AbstractStrings are a pretty handy thing to be able to crunch through
        $f(str::AbstractString) = $f(String(str)) # always crunch UTF-8 repr
        $f(str::String) = $f(codeunits(str))
        $g(key::Vector{UInt8}, str::AbstractString) = $g(key, String(str))
        $g(key::Vector{UInt8}, str::String) = $g(key, codeunits(str))

        """
            $($f)(io::IO)

        Hash data from io using $($f) algorithm from io.
        """
        function $f(io::IO, chunk_size=4*1024)
            ctx = $ctx()
            buff = Vector{UInt8}(undef, chunk_size)
            while !eof(io)
                num_read = readbytes!(io, buff)
                update!(ctx, buff, num_read)
            end
            return digest!(ctx)
        end
        function $g(key::Vector{UInt8}, io::IO, chunk_size=4*1024)
            ctx = HMAC_CTX($ctx(), key)
            buff = Vector{UInt8}(undef, chunk_size)
            while !eof(io)
                num_read = readbytes!(io, buff)
                update!(ctx, buff, num_read)
            end
            return digest!(ctx)
        end
    end
end

end #module SHA

D stdlib/SHA/src/base_functions.jl => stdlib/SHA/src/base_functions.jl +0 -42
@@ 1,42 0,0 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

# THE SIX LOGICAL FUNCTIONS
#
# Bit shifting and rotation (used by the six SHA-XYZ logical functions:
#
#   NOTE:  The naming of R and S appears backwards here (R is a SHIFT and
#   S is a ROTATION) because the SHA2-256/384/512 description document
#   (see http://csrc.nist.gov/cryptval/shs/sha256-384-512.pdf) uses this
#   same "backwards" definition.

# 32-bit Rotate-right (equivalent to S32 in SHA-256) and rotate-left
rrot(b,x,width) = ((x >> b) | (x << (width - b)))
lrot(b,x,width) = ((x << b) | (x >> (width - b)))

# Shift-right (used in SHA-256, SHA-384, and SHA-512):
R(b,x)   = (x >> b)
# 32-bit Rotate-right (used in SHA-256):
S32(b,x) = rrot(b,x,32)
# 64-bit Rotate-right (used in SHA-384 and SHA-512):
S64(b,x) = rrot(b,x,64)
# 64-bit Rotate-left (used in SHA3)
L64(b,x) = lrot(b,x,64)

# Two of six logical functions used in SHA-256, SHA-384, and SHA-512:
Ch(x,y,z)  = ((x & y) ⊻ (~x & z))
Maj(x,y,z) = ((x & y) ⊻ (x & z) ⊻ (y & z))

# Four of six logical functions used in SHA-256:
Sigma0_256(x) = (S32(2,  UInt32(x)) ⊻ S32(13, UInt32(x)) ⊻ S32(22, UInt32(x)))
Sigma1_256(x) = (S32(6,  UInt32(x)) ⊻ S32(11, UInt32(x)) ⊻ S32(25, UInt32(x)))
sigma0_256(x) = (S32(7,  UInt32(x)) ⊻ S32(18, UInt32(x)) ⊻ R(3 ,   UInt32(x)))
sigma1_256(x) = (S32(17, UInt32(x)) ⊻ S32(19, UInt32(x)) ⊻ R(10,   UInt32(x)))

# Four of six logical functions used in SHA-384 and SHA-512:
Sigma0_512(x) = (S64(28, UInt64(x)) ⊻ S64(34, UInt64(x)) ⊻ S64(39, UInt64(x)))
Sigma1_512(x) = (S64(14, UInt64(x)) ⊻ S64(18, UInt64(x)) ⊻ S64(41, UInt64(x)))
sigma0_512(x) = (S64( 1, UInt64(x)) ⊻ S64( 8, UInt64(x)) ⊻ R( 7,   UInt64(x)))
sigma1_512(x) = (S64(19, UInt64(x)) ⊻ S64(61, UInt64(x)) ⊻ R( 6,   UInt64(x)))

# Let's be able to bswap arrays of these types as well
bswap!(x::Vector{<:Integer}) = map!(bswap, x, x)

D stdlib/SHA/src/common.jl => stdlib/SHA/src/common.jl +0 -116
@@ 1,116 0,0 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

# Common update and digest functions which work across SHA1 and SHA2

# update! takes in variable-length data, buffering it into blocklen()-sized pieces,
# calling transform!() when necessary to update the internal hash state.
"""
    update!(context, data[, datalen])

Update the SHA context with the bytes in data. See also [`digest!`](@ref) for
finalizing the hash.

# Examples
```julia-repl
julia> ctx = SHA1_CTX()
SHA1 hash state

julia> update!(ctx, b"data to to be hashed")
```
"""
function update!(context::T, data::U, datalen=length(data)) where {T<:SHA_CTX, U<:AbstractBytes}
    # We need to do all our arithmetic in the proper bitwidth
    UIntXXX = typeof(context.bytecount)

    # Process as many complete blocks as possible
    0 ≤ datalen ≤ length(data) || throw(BoundsError(data, firstindex(data)+datalen-1))
    len = convert(UIntXXX, datalen)
    data_idx = convert(UIntXXX, firstindex(data)-1)
    usedspace = context.bytecount % blocklen(T)
    while len - data_idx + usedspace >= blocklen(T)
        # Fill up as much of the buffer as we can with the data given us
        copyto!(context.buffer, usedspace + 1, data, data_idx + 1, blocklen(T) - usedspace)

        transform!(context)
        context.bytecount += blocklen(T) - usedspace
        data_idx += blocklen(T) - usedspace
        usedspace = convert(UIntXXX, 0)
    end

    # There is less than a complete block left, but we need to save the leftovers into context.buffer:
    if len > data_idx
        copyto!(context.buffer, usedspace + 1, data, data_idx + 1, len - data_idx)
        context.bytecount += len - data_idx
    end
end

# Pad the remainder leaving space for the bitcount
function pad_remainder!(context::T) where T<:SHA_CTX
    usedspace = context.bytecount % blocklen(T)
    # If we have anything in the buffer still, pad and transform that data
    if usedspace > 0
        # Begin padding with a 1 bit:
        context.buffer[usedspace+1] = 0x80
        usedspace += 1

        # If we have room for the bitcount, then pad up to the short blocklen
        if usedspace <= short_blocklen(T)
            for i = 1:(short_blocklen(T) - usedspace)
                context.buffer[usedspace + i] = 0x0
            end
        else
            # Otherwise, pad out this entire block, transform it, then pad up to short blocklen
            for i = 1:(blocklen(T) - usedspace)
                context.buffer[usedspace + i] = 0x0
            end
            transform!(context)
            for i = 1:short_blocklen(T)
                context.buffer[i] = 0x0
            end
        end
    else
        # If we don't have anything in the buffer, pad an entire shortbuffer
        context.buffer[1] = 0x80
        for i = 2:short_blocklen(T)
            context.buffer[i] = 0x0
        end
    end
end


# Clear out any saved data in the buffer, append total bitlength, and return our precious hash!
# Note: SHA3_CTX has a more specialised method
"""
    digest!(context)

Finalize the SHA context and return the hash as array of bytes (Array{Uint8, 1}).

# Examples
```julia-repl
julia> ctx = SHA1_CTX()
SHA1 hash state

julia> update!(ctx, b"data to to be hashed")

julia> digest!(ctx)
20-element Array{UInt8,1}:
 0x83
 0xe4
 0x89
 0xf5
```
"""
function digest!(context::T) where T<:SHA_CTX
    pad_remainder!(context)
    # Store the length of the input data (in bits) at the end of the padding
    bitcount_idx = div(short_blocklen(T), sizeof(context.bytecount)) + 1
    pbuf = Ptr{typeof(context.bytecount)}(pointer(context.buffer))
    unsafe_store!(pbuf, bswap(context.bytecount * 8), bitcount_idx)

    # Final transform:
    transform!(context)

    # Return the digest
    return reinterpret(UInt8, bswap!(context.state))[1:digestlen(T)]
end

D stdlib/SHA/src/constants.jl => stdlib/SHA/src/constants.jl +0 -131
@@ 1,131 0,0 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

# SHA initial hash values and constants

# Hash constant words K for SHA1
const K1 = UInt32[
    0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6
]

# Initial hash value H for SHA1
const SHA1_initial_hash_value = UInt32[
    0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476, 0xc3d2e1f0
]



# Hash constant words K for SHA-256:
const K256 = UInt32[
    0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
    0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
    0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
    0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
    0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
    0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
    0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
    0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
    0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
    0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
    0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
    0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
    0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
    0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
    0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
    0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
]

# Initial hash value H for SHA-224:
const SHA2_224_initial_hash_value = UInt32[
    0xc1059ed8, 0x367cd507, 0x3070dd17, 0xf70e5939,
    0xffc00b31, 0x68581511, 0x64f98fa7, 0xbefa4fa4
]


const SHA2_256_initial_hash_value = UInt32[
    0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
    0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19
]

# Hash constant words K for SHA-384 and SHA-512:
const K512 = UInt64[
    0x428a2f98d728ae22, 0x7137449123ef65cd,
    0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc,
    0x3956c25bf348b538, 0x59f111f1b605d019,
    0x923f82a4af194f9b, 0xab1c5ed5da6d8118,
    0xd807aa98a3030242, 0x12835b0145706fbe,
    0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2,
    0x72be5d74f27b896f, 0x80deb1fe3b1696b1,
    0x9bdc06a725c71235, 0xc19bf174cf692694,
    0xe49b69c19ef14ad2, 0xefbe4786384f25e3,
    0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65,
    0x2de92c6f592b0275, 0x4a7484aa6ea6e483,
    0x5cb0a9dcbd41fbd4, 0x76f988da831153b5,
    0x983e5152ee66dfab, 0xa831c66d2db43210,
    0xb00327c898fb213f, 0xbf597fc7beef0ee4,
    0xc6e00bf33da88fc2, 0xd5a79147930aa725,
    0x06ca6351e003826f, 0x142929670a0e6e70,
    0x27b70a8546d22ffc, 0x2e1b21385c26c926,
    0x4d2c6dfc5ac42aed, 0x53380d139d95b3df,
    0x650a73548baf63de, 0x766a0abb3c77b2a8,
    0x81c2c92e47edaee6, 0x92722c851482353b,
    0xa2bfe8a14cf10364, 0xa81a664bbc423001,
    0xc24b8b70d0f89791, 0xc76c51a30654be30,
    0xd192e819d6ef5218, 0xd69906245565a910,
    0xf40e35855771202a, 0x106aa07032bbd1b8,
    0x19a4c116b8d2d0c8, 0x1e376c085141ab53,
    0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8,
    0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb,
    0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3,
    0x748f82ee5defb2fc, 0x78a5636f43172f60,
    0x84c87814a1f0ab72, 0x8cc702081a6439ec,
    0x90befffa23631e28, 0xa4506cebde82bde9,
    0xbef9a3f7b2c67915, 0xc67178f2e372532b,
    0xca273eceea26619c, 0xd186b8c721c0c207,
    0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178,
    0x06f067aa72176fba, 0x0a637dc5a2c898a6,
    0x113f9804bef90dae, 0x1b710b35131c471b,
    0x28db77f523047d84, 0x32caab7b40c72493,
    0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c,
    0x4cc5d4becb3e42b6, 0x597f299cfc657e2a,
    0x5fcb6fab3ad6faec, 0x6c44198c4a475817
]

# Initial hash value H for SHA-384
const SHA2_384_initial_hash_value = UInt64[
    0xcbbb9d5dc1059ed8, 0x629a292a367cd507,
    0x9159015a3070dd17, 0x152fecd8f70e5939,
    0x67332667ffc00b31, 0x8eb44a8768581511,
    0xdb0c2e0d64f98fa7, 0x47b5481dbefa4fa4
]

# Initial hash value H for SHA-512
const SHA2_512_initial_hash_value = UInt64[
    0x6a09e667f3bcc908, 0xbb67ae8584caa73b,
    0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1,
    0x510e527fade682d1, 0x9b05688c2b3e6c1f,
    0x1f83d9abfb41bd6b, 0x5be0cd19137e2179
]

# Round constants for SHA3 rounds
const SHA3_ROUND_CONSTS = UInt64[
    0x0000000000000001, 0x0000000000008082, 0x800000000000808a,
    0x8000000080008000, 0x000000000000808b, 0x0000000080000001,
    0x8000000080008081, 0x8000000000008009, 0x000000000000008a,
    0x0000000000000088, 0x0000000080008009, 0x000000008000000a,
    0x000000008000808b, 0x800000000000008b, 0x8000000000008089,
    0x8000000000008003, 0x8000000000008002, 0x8000000000000080,
    0x000000000000800a, 0x800000008000000a, 0x8000000080008081,
    0x8000000000008080, 0x0000000080000001, 0x8000000080008008
]

# Rotation constants for SHA3 rounds
const SHA3_ROTC = UInt64[
    1,  3,  6,  10, 15, 21, 28, 36, 45, 55, 2,  14,
    27, 41, 56, 8,  25, 43, 62, 18, 39, 61, 20, 44
]

# Permutation indices for SHA3 rounds (+1'ed so as to work with julia's 1-based indexing)
const SHA3_PILN = Int[
    11, 8,  12, 18, 19, 4, 6,  17, 9,  22, 25, 5,
    16, 24, 20, 14, 13, 3, 21, 15, 23, 10,  7,  2
]

D stdlib/SHA/src/hmac.jl => stdlib/SHA/src/hmac.jl +0 -35
@@ 1,35 0,0 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

struct HMAC_CTX{CTX<:SHA_CTX}
    context::CTX
    outer::Vector{UInt8}

    function HMAC_CTX(ctx::CTX, key::Vector{UInt8}, blocksize::Integer=blocklen(CTX)) where CTX
        if length(key) > blocksize
            _ctx = CTX()
            update!(_ctx, key)
            key = digest!(_ctx)
        end

        pad = blocksize - length(key)

        if pad > 0
            key = [key; fill(0x00, pad)]
        end

        update!(ctx, key .⊻ 0x36)
        new{CTX}(ctx, key .⊻ 0x5c)
    end
end

function update!(ctx::HMAC_CTX, data, datalen=length(data))
    update!(ctx.context, data, datalen)
end

function digest!(ctx::HMAC_CTX{CTX}) where CTX
    digest = digest!(ctx.context)
    _ctx = CTX()
    update!(_ctx, ctx.outer)
    update!(_ctx, digest)
    digest!(_ctx)
end

D stdlib/SHA/src/sha1.jl => stdlib/SHA/src/sha1.jl +0 -95
@@ 1,95 0,0 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

# Nonlinear functions, in order to encourage inlining, these sadly are not an array of lambdas
function Round0(b,c,d)
    return UInt32((b & c) | (~b & d))
end

function Round1And3(b,c,d)
    return UInt32(b ⊻ c ⊻ d)
end

function Round2(b,c,d)
    return UInt32((b & c) | (b & d) | (c & d))
end

function transform!(context::SHA1_CTX)
    # Buffer is 16 elements long, we expand to 80
    pbuf = buffer_pointer(context)
    for i in 1:16
        context.W[i] = bswap(unsafe_load(pbuf, i))
    end

    # First round of expansions
    for i in 17:32
        @inbounds begin
            context.W[i] = lrot(1, context.W[i-3] ⊻ context.W[i-8] ⊻ context.W[i-14] ⊻ context.W[i-16], 32)
        end
    end

    # Second round of expansions (possibly 4-way SIMD-able)
    for i in 33:80
        @inbounds begin
            context.W[i] = lrot(2, context.W[i-6] ⊻ context.W[i-16] ⊻ context.W[i-28] ⊻ context.W[i-32], 32)
        end
    end

    # Initialize registers with the previous intermediate values (our state)
    a = context.state[1]
    b = context.state[2]
    c = context.state[3]
    d = context.state[4]
    e = context.state[5]

    # Run our rounds, manually separated into the four rounds, unfortunately using an array of lambdas
    # really kills performance and causes a huge number of allocations, so we make it easy on the compiler
    for i = 1:20
        @inbounds begin
            temp = UInt32(lrot(5, a, 32) + Round0(b,c,d) + e + context.W[i] + K1[1])
            e = d
            d = c
            c = lrot(30, b, 32)
            b = a
            a = temp
        end
    end

    for i = 21:40
        @inbounds begin
            temp = UInt32(lrot(5, a, 32) + Round1And3(b,c,d) + e + context.W[i] + K1[2])
            e = d
            d = c
            c = lrot(30, b, 32)
            b = a
            a = temp
        end
    end

    for i = 41:60
        @inbounds begin
            temp = UInt32(lrot(5, a, 32) + Round2(b,c,d) + e + context.W[i] + K1[3])
            e = d
            d = c
            c = lrot(30, b, 32)
            b = a
            a = temp
        end
    end

    for i = 61:80
        @inbounds begin
            temp = UInt32(lrot(5, a, 32) + Round1And3(b,c,d) + e + context.W[i] + K1[4])
            e = d
            d = c
            c = lrot(30, b, 32)
            b = a
            a = temp
        end
    end

    context.state[1] += a
    context.state[2] += b
    context.state[3] += c
    context.state[4] += d
    context.state[5] += e
end

D stdlib/SHA/src/sha2.jl => stdlib/SHA/src/sha2.jl +0 -136
@@ 1,136 0,0 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

function transform!(context::T) where {T<:Union{SHA2_224_CTX,SHA2_256_CTX}}
    pbuf = buffer_pointer(context)
    # Initialize registers with the previous intermediate values (our state)
    a = context.state[1]
    b = context.state[2]
    c = context.state[3]
    d = context.state[4]
    e = context.state[5]
    f = context.state[6]
    g = context.state[7]
    h = context.state[8]

    # Run initial rounds
    for j = 1:16
        @inbounds begin
            # We bitswap every input byte
            v = bswap(unsafe_load(pbuf, j))
            unsafe_store!(pbuf, v, j)

            # Apply the SHA-256 compression function to update a..h
            T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + v
            T2 = Sigma0_256(a) + Maj(a, b, c)
            h = g
            g = f
            f = e
            e = UInt32(d + T1)
            d = c
            c = b
            b = a
            a = UInt32(T1 + T2)
        end
    end

    for j = 17:64
        @inbounds begin
            # Implicit message block expansion:
            s0 = unsafe_load(pbuf, mod1(j + 1, 16))
            s0 = sigma0_256(s0)
            s1 = unsafe_load(pbuf, mod1(j + 14, 16))
            s1 = sigma1_256(s1)

            # Apply the SHA-256 compression function to update a..h
            v = unsafe_load(pbuf, mod1(j, 16)) + s1 + unsafe_load(pbuf, mod1(j + 9, 16)) + s0
            unsafe_store!(pbuf, v, mod1(j, 16))
            T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + v
            T2 = Sigma0_256(a) + Maj(a, b, c)
            h = g
            g = f
            f = e
            e = UInt32(d + T1)
            d = c
            c = b
            b = a
            a = UInt32(T1 + T2)
        end
    end

    # Compute the current intermediate hash value
    context.state[1] += a
    context.state[2] += b
    context.state[3] += c
    context.state[4] += d
    context.state[5] += e
    context.state[6] += f
    context.state[7] += g
    context.state[8] += h
end


function transform!(context::Union{SHA2_384_CTX,SHA2_512_CTX})
    pbuf = buffer_pointer(context)
    # Initialize registers with the prev. intermediate value
    a = context.state[1]
    b = context.state[2]
    c = context.state[3]
    d = context.state[4]
    e = context.state[5]
    f = context.state[6]
    g = context.state[7]
    h = context.state[8]

    for j = 1:16
        @inbounds begin
            v = bswap(unsafe_load(pbuf, j))
            unsafe_store!(pbuf, v, j)

            # Apply the SHA-512 compression function to update a..h
            T1 = h + Sigma1_512(e) + Ch(e, f, g) + K512[j] + v
            T2 = Sigma0_512(a) + Maj(a, b, c)
            h = g
            g = f
            f = e
            e = d + T1
            d = c
            c = b
            b = a
            a = T1 + T2
        end
    end

    for j = 17:80
        @inbounds begin
            # Implicit message block expansion:
            s0 = unsafe_load(pbuf, mod1(j + 1, 16))
            s0 = sigma0_512(s0)
            s1 = unsafe_load(pbuf, mod1(j + 14, 16))
            s1 = sigma1_512(s1)

            # Apply the SHA-512 compression function to update a..h
            v = unsafe_load(pbuf, mod1(j, 16)) + s1 + unsafe_load(pbuf, mod1(j + 9, 16)) + s0
            unsafe_store!(pbuf, v, mod1(j, 16))
            T1 = h + Sigma1_512(e) + Ch(e, f, g) + K512[j] + v
            T2 = Sigma0_512(a) + Maj(a, b, c)
            h = g
            g = f
            f = e
            e = d + T1
            d = c
            c = b
            b = a
            a = T1 + T2
        end
    end

    # Compute the current intermediate hash value
    context.state[1] += a
    context.state[2] += b
    context.state[3] += c
    context.state[4] += d
    context.state[5] += e
    context.state[6] += f
    context.state[7] += g
    context.state[8] += h
end

D stdlib/SHA/src/sha3.jl => stdlib/SHA/src/sha3.jl +0 -83
@@ 1,83 0,0 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

function transform!(context::T) where {T<:SHA3_CTX}
    # First, update state with buffer
    pbuf = Ptr{eltype(context.state)}(pointer(context.buffer))
    for idx in 1:div(blocklen(T),8)
        context.state[idx] = context.state[idx] ⊻ unsafe_load(pbuf, idx)
    end
    bc = context.bc
    state = context.state

    # We always assume 24 rounds
    @inbounds for round in 0:23
        # Theta function
        for i in 1:5
            bc[i] = state[i] ⊻ state[i + 5] ⊻ state[i + 10] ⊻ state[i + 15] ⊻ state[i + 20]
        end

        for i in 0:4
            temp = bc[rem(i + 4, 5) + 1] ⊻ L64(1, bc[rem(i + 1, 5) + 1])
            j = 0
            while j <= 20
                state[Int(i + j + 1)] = state[i + j + 1] ⊻ temp
                j += 5
            end
        end

        # Rho Pi
        temp = state[2]
        for i in 1:24
            j = SHA3_PILN[i]
            bc[1] = state[j]
            state[j] = L64(SHA3_ROTC[i], temp)
            temp = bc[1]
        end

        # Chi
        j = 0
        while j <= 20
            for i in 1:5
                bc[i] = state[i + j]
            end
            for i in 0:4
                state[j + i + 1] = state[j + i + 1] ⊻ (~bc[rem(i + 1, 5) + 1] & bc[rem(i + 2, 5) + 1])
            end
            j += 5
        end

        # Iota
        state[1] = state[1] ⊻ SHA3_ROUND_CONSTS[round+1]
    end

    return context.state
end



# Finalize data in the buffer, append total bitlength, and return our precious hash!
function digest!(context::T) where {T<:SHA3_CTX}
    usedspace = context.bytecount % blocklen(T)
    # If we have anything in the buffer still, pad and transform that data
    if usedspace < blocklen(T) - 1
        # Begin padding with a 0x06
        context.buffer[usedspace+1] = 0x06
        # Fill with zeros up until the last byte
        context.buffer[usedspace+2:end-1] .= 0x00
        # Finish it off with a 0x80
        context.buffer[end] = 0x80
    else
        # Otherwise, we have to add on a whole new buffer just for the zeros and 0x80
        context.buffer[end] = 0x06
        transform!(context)

        context.buffer[1:end-1] = 0x0
        context.buffer[end] = 0x80
    end

    # Final transform:
    transform!(context)

    # Return the digest
    return reinterpret(UInt8, context.state)[1:digestlen(T)]
end

D stdlib/SHA/src/types.jl => stdlib/SHA/src/types.jl +0 -230
@@ 1,230 0,0 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

# Type hierarchy to aid in splitting up of SHA2 algorithms
# as SHA224/256 are similar, and SHA-384/512 are similar
abstract type SHA_CTX end
abstract type SHA2_CTX <: SHA_CTX end
abstract type SHA3_CTX <: SHA_CTX end
import Base: copy

# We derive SHA1_CTX straight from SHA_CTX since it doesn't have a
# family of types like SHA2 or SHA3 do
mutable struct SHA1_CTX <: SHA_CTX
    state::Array{UInt32,1}
    bytecount::UInt64
    buffer::Array{UInt8,1}
    W::Array{UInt32,1}
end

# SHA2 224/256/384/512-bit Context Structures
mutable struct SHA2_224_CTX <: SHA2_CTX
    state::Array{UInt32,1}
    bytecount::UInt64
    buffer::Array{UInt8,1}
end

mutable struct SHA2_256_CTX <: SHA2_CTX
    state::Array{UInt32,1}
    bytecount::UInt64
    buffer::Array{UInt8,1}
end

mutable struct SHA2_384_CTX <: SHA2_CTX
    state::Array{UInt64,1}
    bytecount::UInt128
    buffer::Array{UInt8,1}
end

mutable struct SHA2_512_CTX <: SHA2_CTX
    state::Array{UInt64,1}
    bytecount::UInt128
    buffer::Array{UInt8,1}
end

function Base.getproperty(ctx::SHA2_CTX, fieldname::Symbol)
    if fieldname === :state
        return getfield(ctx, :state)::Union{Vector{UInt32},Vector{UInt64}}
    elseif fieldname === :bytecount
        return getfield(ctx, :bytecount)::Union{UInt64,UInt128}
    elseif fieldname === :buffer
        return getfield(ctx, :buffer)::Vector{UInt8}
    elseif fieldname === :W
        return getfield(ctx, :W)::Vector{UInt32}
    else
        error("SHA2_CTX has no field ", fieldname)
    end
end


# Typealias common nicknames for SHA2 family of functions
const SHA224_CTX = SHA2_224_CTX
const SHA256_CTX = SHA2_256_CTX
const SHA384_CTX = SHA2_384_CTX
const SHA512_CTX = SHA2_512_CTX


# SHA3 224/256/384/512-bit context structures
mutable struct SHA3_224_CTX <: SHA3_CTX
    state::Array{UInt64,1}
    bytecount::UInt128
    buffer::Array{UInt8,1}
    bc::Array{UInt64,1}
end
mutable struct SHA3_256_CTX <: SHA3_CTX
    state::Array{UInt64,1}
    bytecount::UInt128
    buffer::Array{UInt8,1}
    bc::Array{UInt64,1}
end
mutable struct SHA3_384_CTX <: SHA3_CTX
    state::Array{UInt64,1}
    bytecount::UInt128
    buffer::Array{UInt8,1}
    bc::Array{UInt64,1}
end
mutable struct SHA3_512_CTX <: SHA3_CTX
    state::Array{UInt64,1}
    bytecount::UInt128
    buffer::Array{UInt8,1}
    bc::Array{UInt64,1}
end

function Base.getproperty(ctx::SHA3_CTX, fieldname::Symbol)
    if fieldname === :state
        return getfield(ctx, :state)::Vector{UInt64}
    elseif fieldname === :bytecount
        return getfield(ctx, :bytecount)::UInt128
    elseif fieldname === :buffer
        return getfield(ctx, :buffer)::Vector{UInt8}
    elseif fieldname === :bc
        return getfield(ctx, :bc)::Vector{UInt64}
    else
        error("type ", typeof(ctx), " has no field ", fieldname)
    end
end

# Define constants via functions so as not to bloat context objects.  Yay dispatch!

# Digest lengths for SHA1, SHA2 and SHA3.  This is easy to figure out from the typename
digestlen(::Type{SHA1_CTX}) = 20
digestlen(::Type{SHA2_224_CTX}) = 28
digestlen(::Type{SHA3_224_CTX}) = 28
digestlen(::Type{SHA2_256_CTX}) = 32
digestlen(::Type{SHA3_256_CTX}) = 32
digestlen(::Type{SHA2_384_CTX}) = 48
digestlen(::Type{SHA3_384_CTX}) = 48
digestlen(::Type{SHA2_512_CTX}) = 64
digestlen(::Type{SHA3_512_CTX}) = 64

# SHA1 and SHA2 have differing element types for the internal state objects
state_type(::Type{SHA1_CTX}) = UInt32
state_type(::Type{SHA2_224_CTX}) = UInt32
state_type(::Type{SHA2_256_CTX}) = UInt32
state_type(::Type{SHA2_384_CTX}) = UInt64
state_type(::Type{SHA2_512_CTX}) = UInt64
state_type(::Type{SHA3_CTX}) = UInt64

# blocklen is the number of bytes of data processed by the transform!() function at once
blocklen(::Type{SHA1_CTX}) = UInt64(64)
blocklen(::Type{SHA2_224_CTX}) = UInt64(64)
blocklen(::Type{SHA2_256_CTX}) = UInt64(64)
blocklen(::Type{SHA2_384_CTX}) = UInt64(128)
blocklen(::Type{SHA2_512_CTX}) = UInt64(128)

blocklen(::Type{SHA3_224_CTX}) = UInt64(25*8 - 2*digestlen(SHA3_224_CTX))
blocklen(::Type{SHA3_256_CTX}) = UInt64(25*8 - 2*digestlen(SHA3_256_CTX))
blocklen(::Type{SHA3_384_CTX}) = UInt64(25*8 - 2*digestlen(SHA3_384_CTX))
blocklen(::Type{SHA3_512_CTX}) = UInt64(25*8 - 2*digestlen(SHA3_512_CTX))


# short_blocklen is the size of a block minus the width of bytecount
short_blocklen(::Type{T}) where {T<:SHA_CTX} = blocklen(T) - 2*sizeof(state_type(T))

# Once the "blocklen" methods are defined, we can define our outer constructors for SHA types:

"""
    SHA2_224_CTX()

Construct an empty SHA2_224 context.
"""
SHA2_224_CTX() = SHA2_224_CTX(copy(SHA2_224_initial_hash_value), 0, zeros(UInt8, blocklen(SHA2_224_CTX)))
"""
    SHA2_256_CTX()

Construct an empty SHA2_256 context.
"""
SHA2_256_CTX() = SHA2_256_CTX(copy(SHA2_256_initial_hash_value), 0, zeros(UInt8, blocklen(SHA2_256_CTX)))
"""
    SHA2_384()

Construct an empty SHA2_384 context.
"""
SHA2_384_CTX() = SHA2_384_CTX(copy(SHA2_384_initial_hash_value), 0, zeros(UInt8, blocklen(SHA2_384_CTX)))
"""
    SHA2_512_CTX()

Construct an empty SHA2_512 context.
"""
SHA2_512_CTX() = SHA2_512_CTX(copy(SHA2_512_initial_hash_value), 0, zeros(UInt8, blocklen(SHA2_512_CTX)))

"""
    SHA3_224_CTX()

Construct an empty SHA3_224 context.
"""
SHA3_224_CTX() = SHA3_224_CTX(zeros(UInt64, 25), 0, zeros(UInt8, blocklen(SHA3_224_CTX)), Vector{UInt64}(undef, 5))
"""
    SHA3_256_CTX()

Construct an empty SHA3_256 context.
"""
SHA3_256_CTX() = SHA3_256_CTX(zeros(UInt64, 25), 0, zeros(UInt8, blocklen(SHA3_256_CTX)), Vector{UInt64}(undef, 5))
"""
    SHA3_384_CTX()

Construct an empty SHA3_384 context.
"""
SHA3_384_CTX() = SHA3_384_CTX(zeros(UInt64, 25), 0, zeros(UInt8, blocklen(SHA3_384_CTX)), Vector{UInt64}(undef, 5))
"""
    SHA3_512_CTX()

Construct an empty SHA3_512 context.
"""
SHA3_512_CTX() = SHA3_512_CTX(zeros(UInt64, 25), 0, zeros(UInt8, blocklen(SHA3_512_CTX)), Vector{UInt64}(undef, 5))

# Nickname'd outer constructor methods for SHA2
const SHA224_CTX = SHA2_224_CTX
const SHA256_CTX = SHA2_256_CTX
const SHA384_CTX = SHA2_384_CTX
const SHA512_CTX = SHA2_512_CTX

# SHA1 is special; he needs extra workspace
"""
    SHA1_CTX()

Construct an empty SHA1 context.
"""
SHA1_CTX() = SHA1_CTX(copy(SHA1_initial_hash_value), 0, zeros(UInt8, blocklen(SHA1_CTX)), Vector{UInt32}(undef, 80))


# Copy functions
copy(ctx::T) where {T<:SHA1_CTX} = T(copy(ctx.state), ctx.bytecount, copy(ctx.buffer), copy(ctx.W))
copy(ctx::T) where {T<:SHA2_CTX} = T(copy(ctx.state), ctx.bytecount, copy(ctx.buffer))
copy(ctx::T) where {T<:SHA3_CTX} = T(copy(ctx.state), ctx.bytecount, copy(ctx.buffer), Vector{UInt64}(undef, 5))


# Make printing these types a little friendlier
import Base.show
show(io::IO, ::SHA1_CTX) = print(io, "SHA1 hash state")
show(io::IO, ::SHA2_224_CTX) = print(io, "SHA2 224-bit hash state")
show(io::IO, ::SHA2_256_CTX) = print(io, "SHA2 256-bit hash state")
show(io::IO, ::SHA2_384_CTX) = print(io, "SHA2 384-bit hash state")
show(io::IO, ::SHA2_512_CTX) = print(io, "SHA2 512-bit hash state")
show(io::IO, ::SHA3_224_CTX) = print(io, "SHA3 224-bit hash state")
show(io::IO, ::SHA3_256_CTX) = print(io, "SHA3 256-bit hash state")
show(io::IO, ::SHA3_384_CTX) = print(io, "SHA3 384-bit hash state")
show(io::IO, ::SHA3_512_CTX) = print(io, "SHA3 512-bit hash state")


# use our types to define a method to get a pointer to the state buffer
buffer_pointer(ctx::T) where {T<:SHA_CTX} = Ptr{state_type(T)}(pointer(ctx.buffer))

D stdlib/SHA/test/perf.jl => stdlib/SHA/test/perf.jl +0 -47
@@ 1,47 0,0 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

using SHA

if isempty(ARGS)
    error("need file to test sha perf")
elseif !isfile(ARGS[1])
    error("file $(ARGS[1]) does not exist")
end


function do_tests(filepath)
    # test performance
    print("read:    ")
    @time begin
        fh = open(filepath, "r")
        bytes = read(fh)
    end
    GC.gc()

    print("SHA-1:   ")
    sha1(bytes)
    GC.gc()
    @time sha1(bytes)

    print("SHA2-256: ")
    sha256(bytes)
    GC.gc()
    @time sha256(bytes)

    print("SHA2-512: ")
    sha512(bytes)
    GC.gc()
    @time sha512(bytes)

    print("SHA3-256: ")
    sha3_256(bytes)
    GC.gc()
    @time sha3_256(bytes)

    print("SHA3-512: ")
    sha3_512(bytes)
    GC.gc()
    @time sha3_512(bytes)
end

do_tests(ARGS[1])

D stdlib/SHA/test/runtests.jl => stdlib/SHA/test/runtests.jl +0 -304
@@ 1,304 0,0 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

using SHA
using Test

const VERBOSE = false

# Define some data we will run our tests on
lorem = "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
so_many_as_array = repeat([0x61], 1000000)
so_many_as_tuple = ntuple((i) -> 0x61, 1000000)
tempdir = mktempdir()
file = joinpath(tempdir, ".sha")
fIO = open(file, "w")
write(fIO, '\0')
close(fIO)
data = Any["", "test", lorem, file, so_many_as_array, so_many_as_tuple]

# Descriptions of the data, the SHA functions we'll run on the data, etc...
data_desc = ["the empty string", "the string \"test\"", "lorem ipsum",
             "0 file", "one million a's Array", "one million a's Tuple"]
sha_types = Dict(sha1 => SHA.SHA1_CTX,
            sha2_224 => SHA.SHA2_224_CTX, sha2_256 => SHA.SHA2_256_CTX, sha2_384 => SHA.SHA2_384_CTX, sha2_512 => SHA.SHA2_512_CTX,
            sha3_224 => SHA.SHA3_224_CTX, sha3_256 => SHA.SHA3_256_CTX, sha3_384 => SHA.SHA3_384_CTX, sha3_512 => SHA.SHA3_512_CTX)
sha_funcs = [sha1,
             sha2_224, sha2_256, sha2_384, sha2_512,
             sha3_224, sha3_256, sha3_384, sha3_512]
ctxs = [SHA1_CTX,
        SHA2_224_CTX, SHA2_256_CTX, SHA2_384_CTX, SHA2_512_CTX,
        SHA3_224_CTX, SHA3_256_CTX, SHA3_384_CTX, SHA3_512_CTX]
shws = ["SHA1 hash state",
        "SHA2 224-bit hash state", "SHA2 256-bit hash state", "SHA2 384-bit hash state", "SHA2 512-bit hash state",
        "SHA3 224-bit hash state", "SHA3 256-bit hash state", "SHA3 384-bit hash state", "SHA3 512-bit hash state"]

answers = Dict(
sha1 => [
"da39a3ee5e6b4b0d3255bfef95601890afd80709",
"a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
"19afa2a4a37462c7b940a6c4c61363d49c3a35f4",
"5ba93c9db0cff93f52b521d7420e43f6eda2784f",
"34aa973cd4c4daa4f61eeb2bdbad27316534016f",
"34aa973cd4c4daa4f61eeb2bdbad27316534016f"
],
sha2_224 => [
"d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f",
"90a3ed9e32b2aaf4c61c410eb925426119e1a9dc53d4286ade99a809",
"6a0644abcf1e2cecbec2814443dab5f24b7ad8ebb66c75667ab67959",
"fff9292b4201617bdc4d3053fce02734166a683d7d858a7f5f59b073",
"20794655980c91d8bbb4c1ea97618a4bf03f42581948b2ee4ee7ad67",
"20794655980c91d8bbb4c1ea97618a4bf03f42581948b2ee4ee7ad67"
],
sha2_256 => [
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
"2c7c3d5f244f1a40069a32224215e0cf9b42485c99d80f357d76f006359c7a18",
"6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d",
"cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0",
"cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0"
],
sha2_384 => [
"38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b",
"768412320f7b0aa5812fce428dc4706b3cae50e02a64caa16a782249bfe8efc4b7ef1ccb126255d196047dfedf17a0a9",
"63980fd0425cd2c3d8a400ee0f2671ef135db03b947ec1af21b6e28f19c16ca272036469541f4d8e336ac6d1da50580f",
"bec021b4f368e3069134e012c2b4307083d3a9bdd206e24e5f0d86e13d6636655933ec2b413465966817a9c208a11717",
"9d0e1809716474cb086e834e310a4a1ced149e9c00f248527972cec5704c2a5b07b8b3dc38ecc4ebae97ddd87f3d8985",
"9d0e1809716474cb086e834e310a4a1ced149e9c00f248527972cec5704c2a5b07b8b3dc38ecc4ebae97ddd87f3d8985"
],
sha2_512 => [
"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
"ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff",
"f41d92bc9fc1157a0d1387e67f3d0893b70f7039d3d46d8115b5079d45ad601159398c79c281681e2da09bf7d9f8c23b41d1a0a3c5b528a7f2735933a4353194",
"b8244d028981d693af7b456af8efa4cad63d282e19ff14942c246e50d9351d22704a802a71c3580b6370de4ceb293c324a8423342557d4e5c38438f0e36910ee",
"e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973ebde0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b",
"e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973ebde0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b"
],
sha3_224 => [
"6b4e03423667dbb73b6e15454f0eb1abd4597f9a1b078e3f5b5a6bc7",
"3797bf0afbbfca4a7bbba7602a2b552746876517a7f9b7ce2db0ae7b",
"ea5395370949ad8c7d2ca3e7c045ef3306fe3a3f4740de452ef87a28",
"bdd5167212d2dc69665f5a8875ab87f23d5ce7849132f56371a19096",
"d69335b93325192e516a912e6d19a15cb51c6ed5c15243e7a7fd653c",
"d69335b93325192e516a912e6d19a15cb51c6ed5c15243e7a7fd653c"
],
sha3_256 => [
"a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a",
"36f028580bb02cc8272a9a020f4200e346e276ae664e45ee80745574e2f5ab80",
"8c8142d2ca964ab307ace567ddd5764f17ebb76eb8ff25543ab54c14fe2ab139",
"5d53469f20fef4f8eab52b88044ede69c77a6a68a60728609fc4a65ff531e7d0",
"5c8875ae474a3634ba4fd55ec85bffd661f32aca75c6d699d0cdcb6c115891c1",
"5c8875ae474a3634ba4fd55ec85bffd661f32aca75c6d699d0cdcb6c115891c1",
],
sha3_384 => [
"0c63a75b845e4f7d01107d852e4c2485c51a50aaaa94fc61995e71bbee983a2ac3713831264adb47fb6bd1e058d5f004",
"e516dabb23b6e30026863543282780a3ae0dccf05551cf0295178d7ff0f1b41eecb9db3ff219007c4e097260d58621bd",
"eb9fbba3eb916a4efe384b3125f5d03ceb9c5c1b94431ac30fa86c54408b92701ca5d2628cd7113aa5541177ec3ccd1d",
"127677f8b66725bbcb7c3eae9698351ca41e0eb6d66c784bd28dcdb3b5fb12d0c8e840342db03ad1ae180b92e3504933",
"eee9e24d78c1855337983451df97c8ad9eedf256c6334f8e948d252d5e0e76847aa0774ddb90a842190d2c558b4b8340",
"eee9e24d78c1855337983451df97c8ad9eedf256c6334f8e948d252d5e0e76847aa0774ddb90a842190d2c558b4b8340",
],
sha3_512 => [
"a69f73cca23a9ac5c8b567dc185a756e97c982164fe25859e0d1dcc1475c80a615b2123af1f5f94c11e3e9402c3ac558f500199d95b6d3e301758586281dcd26",
"9ece086e9bac491fac5c1d1046ca11d737b92a2b2ebd93f005d7b710110c0a678288166e7fbe796883a4f2e9b3ca9f484f521d0ce464345cc1aec96779149c14",
"3a4318353396a12dfd20442cfce1d8ad4d7e732e85cc56b01b4cf9057a41c8827c0a03c70812e76ace68d776759225c213b4f581aac0dba5dd43b785b1a33fe5",
"7127aab211f82a18d06cf7578ff49d5089017944139aa60d8bee057811a15fb55a53887600a3eceba004de51105139f32506fe5b53e1913bfa6b32e716fe97da",
"3c3a876da14034ab60627c077bb98f7e120a2a5370212dffb3385a18d4f38859ed311d0a9d5141ce9cc5c66ee689b266a8aa18ace8282a0e0db596c90b0a7b87",
"3c3a876da14034ab60627c077bb98f7e120a2a5370212dffb3385a18d4f38859ed311d0a9d5141ce9cc5c66ee689b266a8aa18ace8282a0e0db596c90b0a7b87",
]
)

function describe_hash(T::Type{S}) where {S <: SHA.SHA_CTX}
    if T <: SHA.SHA1_CTX return "SHA1" end
    if T <: SHA.SHA2_CTX return "SHA2-$(SHA.digestlen(T)*8)" end
    if T <: SHA.SHA3_CTX return "SHA3-$(SHA.digestlen(T)*8)" end
end

VERBOSE && println("Loaded hash types: $(join(sort([describe_hash(t[2]) for t in sha_types]), ", ", " and "))")

# First, test processing the data in one go
nerrors = 0
for idx in 1:length(data)
    global nerrors

    desc = data_desc[idx]
    VERBOSE && print("Testing on $desc$(join(["." for z in 1:(34-length(desc))]))")
    nerrors_old = nerrors
    for sha_idx in 1:length(sha_funcs)
        sha_func = sha_funcs[sha_idx]

        if idx == 4
            open(data[idx]) do f
                hash = bytes2hex(sha_func(f))
            end
        else
            hash = bytes2hex(sha_func(data[idx]))
        end

        if hash != answers[sha_func][idx]
            print("\n")
            @warn(
            """
            For $(describe_hash(sha_types[sha_func])) expected:
                $(answers[sha_func][idx])
            Calculated:
                $(hash)
            """)
            nerrors += 1
        else
            VERBOSE && print(".")
        end
    end
    VERBOSE && println("Done! [$(nerrors - nerrors_old) errors]")
end

# Do another test on the "so many a's" data where we chunk up the data into
# two chunks, (sized appropriately to AVOID overflow from one update to another)
# in order to test multiple update!() calls
VERBOSE && print("Testing on one million a's (chunked properly)")
nerrors_old = nerrors
for sha_idx in 1:length(sha_funcs)
    global nerrors

    ctx = sha_types[sha_funcs[sha_idx]]()
    SHA.update!(ctx, so_many_as_array[1:2*SHA.blocklen(typeof(ctx))])
    SHA.update!(ctx, so_many_as_array[2*SHA.blocklen(typeof(ctx))+1:end])
    hash = bytes2hex(SHA.digest!(ctx))
    if hash != answers[sha_funcs[sha_idx]][end]
        print("\n")
        @warn(
        """
        For $(describe_hash(sha_types[sha_funcs[sha_idx]])) expected:
            $(answers[sha_funcs[sha_idx]][end-1])
        Calculated:
            $(hash)
        """)
        nerrors += 1
    else
        VERBOSE && print(".")
    end
end
VERBOSE && println("Done! [$(nerrors - nerrors_old) errors]")

# Do another test on the "so many a's" data where we chunk up the data into
# three chunks, (sized appropriately to CAUSE overflow from one update to another)
# in order to test multiple update!() calls as well as the overflow codepaths
VERBOSE && print("Testing on one million a's (chunked clumsily)")
nerrors_old = nerrors
for sha_idx in 1:length(sha_funcs)
    global nerrors
    ctx = sha_types[sha_funcs[sha_idx]]()

    # Get indices awkwardly placed for the blocklength of this hash type
    idx0 = round(Int, 0.3*SHA.blocklen(typeof(ctx)))
    idx1 = round(Int, 1.7*SHA.blocklen(typeof(ctx)))
    idx2 = round(Int, 2.6*SHA.blocklen(typeof(ctx)))

    # Feed data in according to our dastardly blocking scheme
    SHA.update!(ctx, so_many_as_array[0      + 1:1*idx0])
    SHA.update!(ctx, so_many_as_array[1*idx0 + 1:2*idx0])
    SHA.update!(ctx, so_many_as_array[2*idx0 + 1:3*idx0])
    SHA.update!(ctx, so_many_as_array[3*idx0 + 1:4*idx0])
    SHA.update!(ctx, so_many_as_array[4*idx0 + 1:idx1])
    SHA.update!(ctx, so_many_as_array[idx1 + 1:idx2])
    SHA.update!(ctx, so_many_as_array[idx2 + 1:end])

    # Ensure the hash is the appropriate one
    hash = bytes2hex(SHA.digest!(ctx))
    if hash != answers[sha_funcs[sha_idx]][end]
        print("\n")
        @warn(
        """
        For $(describe_hash(sha_types[sha_funcs[sha_idx]])) expected:
            $(answers[sha_funcs[sha_idx]][end-1])
        Calculated:
            $(hash)
        """)
        nerrors += 1
    else
        VERBOSE && print(".")
    end
end
VERBOSE && println("Done! [$(nerrors - nerrors_old) errors]")

# test hmac correctness using the examples on [wiki](https://en.wikipedia.org/wiki/Hash-based_message_authentication_code#Examples)
VERBOSE && print("Testing on the hmac functions")
nerrors_old = nerrors
for (key, msg, fun, hash) in (
    (b"", b"", hmac_sha1, "fbdb1d1b18aa6c08324b7d64b71fb76370690e1d"),
    (b"", b"", hmac_sha256, "b613679a0814d9ec772f95d778c35fc5ff1697c493715653c6c712144292c5ad"),
    (b"key", b"The quick brown fox jumps over the lazy dog", hmac_sha1, "de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9"),
    (b"key", b"The quick brown fox jumps over the lazy dog", hmac_sha256, "f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8"),
)
    global nerrors
    digest1 = bytes2hex(fun(Vector(key), Vector(msg)))
    digest2 = bytes2hex(fun(Vector(key), IOBuffer(Vector(msg))))
    if digest1 != hash || digest2 != hash
        print("\n")
        @warn(
        """
        For $fun($(String(key)), $(String(msg))) expected:
            $hash
        Calculated:
            $digest1
            $digest2
        """)
        nerrors += 1
    else
        VERBOSE && print(".")
    end
end
VERBOSE && println("Done! [$(nerrors - nerrors_old) errors]")

replstr(x) = sprint((io, x) -> show(IOContext(io, :limit => true), MIME("text/plain"), x), x)

for idx in 1:length(ctxs)
    global nerrors
    # Part #1: copy
    VERBOSE && print("Testing copy function @ $(ctxs[idx]) ...")
    try
        copy(ctxs[idx]())
    catch
        print("\n")
        @warn("Some weird copy error happened with $(ctxs[idx])")
        nerrors += 1
    end
    VERBOSE && println("Done! [$(nerrors - nerrors_old) errors]")

    # Part #2: show
    VERBOSE && print("Testing show function @ $(ctxs[idx]) ...")
    if replstr(ctxs[idx]()) != shws[idx]
        print("\n")
        @warn("Some weird show error happened with $(ctxs[idx])")
        nerrors += 1
    end
    VERBOSE && println("Done! [$(nerrors - nerrors_old) errors]")
end

# test error if eltype of input is not UInt8
for f in sha_funcs
    global nerrors
    local data = UInt32[0x23467, 0x324775]
    try
        f(data)
    catch ex
        if ex isa MethodError &&
            ex.f === f &&
            ex.args === (data,)
            continue
        end
        rethrow()
    end
    @warn("Non-UInt8 Arrays should fail")
    nerrors += 1
end

# Clean up the I/O mess
rm(file)
rm(tempdir)

if nerrors == 0
    VERBOSE && println("ALL OK")
else
    println("Failed with $nerrors failures")
end
@test nerrors == 0