mirror of https://github.com/schollz/croc.git
use imohash as default as its faster
This commit is contained in:
Zack Scholl
parent
ea115908fc
commit
711690a0f2
2
go.mod
2
go.mod
|
|
@ -1,9 +1,11 @@
|
|||
module github.com/schollz/croc/v6
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575
|
||||
github.com/denisbrodbeck/machineid v1.0.1
|
||||
github.com/fatih/color v1.7.0 // indirect
|
||||
github.com/kalafut/imohash v1.0.0
|
||||
github.com/mattn/go-colorable v0.1.1 // indirect
|
||||
github.com/mattn/go-isatty v0.0.7 // indirect
|
||||
github.com/pkg/errors v0.8.1
|
||||
|
|
|
|||
10
go.sum
10
go.sum
|
|
@ -1,3 +1,7 @@
|
|||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs=
|
||||
github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
@ -7,6 +11,8 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS
|
|||
github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
|
||||
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/kalafut/imohash v1.0.0 h1:LgCJ+p/BwM2HKpOxFopkeddpzVCfm15EtXMroXD1SYE=
|
||||
github.com/kalafut/imohash v1.0.0/go.mod h1:c3RHT80ZAp5C/aYgQI92ZlrOymqkZnRDprU87kg75HI=
|
||||
github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
|
||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
|
|
@ -30,6 +36,8 @@ github.com/schollz/progressbar/v2 v2.12.1 h1:0Ce7IBClG+s3lxXN1Noqwh7aToKGL5a3mnM
|
|||
github.com/schollz/progressbar/v2 v2.12.1/go.mod h1:fBI3onORwtNtwCWJHsrXtjE3QnJOtqIZrvr3rDaF7L0=
|
||||
github.com/schollz/spinner v0.0.0-20180925172146-6bbc5f7804f9 h1:y08o5oQ/slxXE/F0uh5dd8mdVvb+w4NLcNSDSq4c2F0=
|
||||
github.com/schollz/spinner v0.0.0-20180925172146-6bbc5f7804f9/go.mod h1:kCMoQsqzx4MzGJWaALr6tKyCnlrY0kILGLkA1FOiLF4=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
|
|
@ -57,3 +65,5 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
|||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/tylerb/is.v1 v1.1.2 h1:AB/MANFml2ySf+adwcinvajyHvsYltAOD+rb/8njfSU=
|
||||
gopkg.in/tylerb/is.v1 v1.1.2/go.mod h1:9yQB2tyIhZ5oph6Kk5Sq7cJMd9c5Jpa1p3hr9kxzPqo=
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package utils
|
|||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
|
|
@ -9,13 +10,14 @@ import (
|
|||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"bytes"
|
||||
"net"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/cespare/xxhash"
|
||||
"github.com/kalafut/imohash"
|
||||
"github.com/schollz/mnemonicode"
|
||||
)
|
||||
|
||||
|
|
@ -37,8 +39,12 @@ func GetInput(prompt string) string {
|
|||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
// HashFile returns the md5 hash of a file
|
||||
// HashFile returns the hash of a file
|
||||
func HashFile(fname string) (hash256 []byte, err error) {
|
||||
return IMOHashFile(fname)
|
||||
}
|
||||
|
||||
func MD5HashFile(fname string) (hash256 []byte, err error) {
|
||||
f, err := os.Open(fname)
|
||||
if err != nil {
|
||||
return
|
||||
|
|
@ -54,6 +60,30 @@ func HashFile(fname string) (hash256 []byte, err error) {
|
|||
return
|
||||
}
|
||||
|
||||
// IMOHashFile returns imohash
|
||||
func IMOHashFile(fname string) (hash []byte, err error) {
|
||||
b, err := imohash.SumFile(fname)
|
||||
hash = b[:]
|
||||
return
|
||||
}
|
||||
|
||||
// XXHashFile returns the xxhash of a file
|
||||
func XXHashFile(fname string) (hash256 []byte, err error) {
|
||||
f, err := os.Open(fname)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
h := xxhash.New()
|
||||
if _, err = io.Copy(h, f); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
hash256 = h.Sum(nil)
|
||||
return
|
||||
}
|
||||
|
||||
// SHA256 returns sha256 sum
|
||||
func SHA256(s string) string {
|
||||
sha := sha256.New()
|
||||
|
|
@ -112,8 +142,6 @@ func ByteCountDecimal(b int64) string {
|
|||
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp])
|
||||
}
|
||||
|
||||
|
||||
|
||||
// MissingChunks returns the positions of missing chunks.
|
||||
// If file doesn't exist, it returns an empty chunk list (all chunks).
|
||||
// If the file size is not the same as requested, it returns an empty chunk list (all chunks).
|
||||
|
|
|
|||
|
|
@ -2,24 +2,60 @@ package utils
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// Exists reports whether the named file or directory exists.
|
||||
func init() {
|
||||
rand.Seed(0)
|
||||
bigBuff := make([]byte, 75000000)
|
||||
rand.Read(bigBuff)
|
||||
ioutil.WriteFile("bigfile.test", bigBuff, 0666)
|
||||
}
|
||||
|
||||
func BenchmarkMD5(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
MD5HashFile("bigfile.test")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkXXHash(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
XXHashFile("bigfile.test")
|
||||
}
|
||||
}
|
||||
func BenchmarkImoHash(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
IMOHashFile("bigfile.test")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExists(t *testing.T) {
|
||||
assert.True(t, Exists("utils.go"))
|
||||
assert.True(t, Exists("bigfile.test"))
|
||||
assert.False(t, Exists("doesnotexist"))
|
||||
}
|
||||
|
||||
// HashFile returns the md5 hash of a file
|
||||
func TestHashFile(t *testing.T) {
|
||||
b, err := HashFile("utils.go")
|
||||
func TestMD5HashFile(t *testing.T) {
|
||||
b, err := MD5HashFile("bigfile.test")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "314965d4170cd0c70bd2e4f6c977750a", fmt.Sprintf("%x", b))
|
||||
assert.Equal(t, "9fed05acbacbc6a36555c998501c21f6", fmt.Sprintf("%x", b))
|
||||
}
|
||||
|
||||
func TestIMOHashFile(t *testing.T) {
|
||||
b, err := IMOHashFile("bigfile.test")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "c0d1e123a96a598ea801cc503d3db8c0", fmt.Sprintf("%x", b))
|
||||
}
|
||||
|
||||
func TestXXHashFile(t *testing.T) {
|
||||
b, err := XXHashFile("bigfile.test")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "f2da6ee7e75e8324", fmt.Sprintf("%x", b))
|
||||
}
|
||||
|
||||
// SHA256 returns sha256 sum
|
||||
func TestSHA256(t *testing.T) {
|
||||
assert.Equal(t, "09ca7e4eaa6e8ae9c7d261167129184883644d07dfba7cbfbc4c8a2e08360d5b", SHA256("hello, world"))
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue