From e9949ce3d7e7a12c36183164e0355d63e8601047 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Tue, 20 Apr 2021 09:08:02 -0700 Subject: [PATCH 1/6] check file size firsty --- src/croc/croc.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/croc/croc.go b/src/croc/croc.go index deb826b..5a6e6ae 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -1236,7 +1236,13 @@ func (c *Client) updateIfRecipientHasFileInfo() (err error) { if i < c.FilesToTransferCurrentNum { continue } - fileHash, errHash := utils.HashFile(path.Join(fileInfo.FolderRemote, fileInfo.Name)) + recipientFileInfo, errRecipientFile := os.Lstat(path.Join(fileInfo.FolderRemote, fileInfo.Name)) + var errHash error + var fileHash []byte + if errRecipientFile == nil && recipientFileInfo.Size() == fileInfo.Size { + // the file exists, but is same size, so hash it + fileHash, errHash = utils.HashFile(path.Join(fileInfo.FolderRemote, fileInfo.Name)) + } if fileInfo.Size == 0 || fileInfo.Symlink != "" { err = c.createEmptyFileAndFinish(fileInfo, i) if err != nil { @@ -1275,7 +1281,6 @@ func (c *Client) updateIfRecipientHasFileInfo() (err error) { c.LastFolder = newFolder break } - // TODO: print out something about this file already existing } err = c.recipientGetFileReady(finished) return From b655afb5332b948cf326a6e68c5862a6d174a88b Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Tue, 20 Apr 2021 15:32:05 -0700 Subject: [PATCH 2/6] choose hash algorithm --- src/cli/cli.go | 2 ++ src/croc/croc.go | 12 +++++++++--- src/utils/utils.go | 16 ++++++++++++---- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/cli/cli.go b/src/cli/cli.go index 5b2d9a9..bf8a77f 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -198,6 +198,7 @@ func send(c *cli.Context) (err error) { NoCompress: c.Bool("no-compress"), Overwrite: c.Bool("overwrite"), Curve: c.String("curve"), + HashAlgorithm: "xxhash", } if crocOptions.RelayAddress != models.DEFAULT_RELAY { crocOptions.RelayAddress6 = "" @@ -395,6 +396,7 @@ func receive(c *cli.Context) (err error) { IP: c.String("ip"), Overwrite: c.Bool("overwrite"), Curve: c.String("curve"), + HashAlgorithm: "xxhash", } if crocOptions.RelayAddress != models.DEFAULT_RELAY { crocOptions.RelayAddress6 = "" diff --git a/src/croc/croc.go b/src/croc/croc.go index 5a6e6ae..b95671d 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -67,6 +67,7 @@ type Options struct { IP string Overwrite bool Curve string + HashAlgorithm string } // Client holds the state of the croc transfer @@ -149,6 +150,7 @@ type SenderInfo struct { Ask bool SendingText bool NoCompress bool + HashAlgorithm string } // New establishes a new connection for transferring files between two instances. @@ -222,8 +224,7 @@ func (c *Client) sendCollectFiles(options TransferOptions) (err error) { } log.Debugf("%+v", c.FilesToTransfer[i]) } - - c.FilesToTransfer[i].Hash, err = utils.HashFile(fullPath) + c.FilesToTransfer[i].Hash, err = utils.HashFile(fullPath, c.Options.HashAlgorithm) totalFilesSize += fstats.Size() if err != nil { return @@ -769,6 +770,11 @@ func (c *Client) processMessageFileInfo(m message.Message) (done bool, err error } c.Options.SendingText = senderInfo.SendingText c.Options.NoCompress = senderInfo.NoCompress + c.Options.HashAlgorithm = senderInfo.HashAlgorithm + if c.Options.HashAlgorithm == "" { + c.Options.HashAlgorithm = "imohash" + } + log.Debugf("using hash algorithm: %s", c.Options.HashAlgorithm) if c.Options.NoCompress { log.Debug("disabling compression") } @@ -1241,7 +1247,7 @@ func (c *Client) updateIfRecipientHasFileInfo() (err error) { var fileHash []byte if errRecipientFile == nil && recipientFileInfo.Size() == fileInfo.Size { // the file exists, but is same size, so hash it - fileHash, errHash = utils.HashFile(path.Join(fileInfo.FolderRemote, fileInfo.Name)) + fileHash, errHash = utils.HashFile(path.Join(fileInfo.FolderRemote, fileInfo.Name), c.Options.HashAlgorithm) } if fileInfo.Size == 0 || fileInfo.Symlink != "" { err = c.createEmptyFileAndFinish(fileInfo, i) diff --git a/src/utils/utils.go b/src/utils/utils.go index f7b2e24..f7a4166 100644 --- a/src/utils/utils.go +++ b/src/utils/utils.go @@ -43,9 +43,8 @@ func GetInput(prompt string) string { } // HashFile returns the hash of a file or, in case of a symlink, the -// SHA256 hash of its target -// HashFile returns the hash of a file -func HashFile(fname string) (hash256 []byte, err error) { +// SHA256 hash of its target. Takes an argument to specify the algorithm to use. +func HashFile(fname string, algorithm string) (hash256 []byte, err error) { var fstats os.FileInfo fstats, err = os.Lstat(fname) if err != nil { @@ -59,7 +58,16 @@ func HashFile(fname string) (hash256 []byte, err error) { if err != nil { return nil, err } - return IMOHashFile(fname) + switch algorithm { + case "imohash": + return IMOHashFile(fname) + case "md5": + return MD5HashFile(fname) + case "xxhash": + return XXHashFile(fname) + } + err = fmt.Errorf("unspecified algorithm") + return } // MD5HashFile returns MD5 hash From 669aeb377ae05c83501be4bd1342a617ee9aaf94 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Tue, 20 Apr 2021 15:35:38 -0700 Subject: [PATCH 3/6] check if file exists --- src/croc/croc.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/croc/croc.go b/src/croc/croc.go index b95671d..fb872fb 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -1053,6 +1053,7 @@ func (c *Client) updateIfSenderChannelSecured() (err error) { Ask: c.Options.Ask, SendingText: c.Options.SendingText, NoCompress: c.Options.NoCompress, + HashAlgorithm: c.Options.HashAlgorithm, }) if err != nil { log.Error(err) @@ -1261,7 +1262,7 @@ func (c *Client) updateIfRecipientHasFileInfo() (err error) { log.Debugf("%s %+x %+x %+v", fileInfo.Name, fileHash, fileInfo.Hash, errHash) if !bytes.Equal(fileHash, fileInfo.Hash) { log.Debugf("hashes are not equal %x != %x", fileHash, fileInfo.Hash) - if errHash == nil && !c.Options.Overwrite { + if errHash == nil && !c.Options.Overwrite && errRecipientFile == nil { log.Debug("asking to overwrite") ans := utils.GetInput(fmt.Sprintf("\nOverwrite '%s'? (y/n) ", path.Join(fileInfo.FolderRemote, fileInfo.Name))) if strings.TrimSpace(strings.ToLower(ans)) != "y" { From 7ac7be37af71ca0428e255761a304c89dfd1973e Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Tue, 20 Apr 2021 15:36:46 -0700 Subject: [PATCH 4/6] check if file exists --- src/croc/croc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/croc/croc.go b/src/croc/croc.go index fb872fb..2a8d281 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -1239,10 +1239,10 @@ func (c *Client) updateIfRecipientHasFileInfo() (err error) { if _, ok := c.FilesHasFinished[i]; ok { continue } - log.Debugf("checking %+v", fileInfo) if i < c.FilesToTransferCurrentNum { continue } + log.Debugf("checking %+v", fileInfo) recipientFileInfo, errRecipientFile := os.Lstat(path.Join(fileInfo.FolderRemote, fileInfo.Name)) var errHash error var fileHash []byte From 85e7576311add01a4ccd8eb73185db90a9e2d5e6 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Tue, 20 Apr 2021 15:44:28 -0700 Subject: [PATCH 5/6] fix --- src/croc/croc.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/croc/croc.go b/src/croc/croc.go index 2a8d281..1c41c6d 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -225,6 +225,7 @@ func (c *Client) sendCollectFiles(options TransferOptions) (err error) { log.Debugf("%+v", c.FilesToTransfer[i]) } c.FilesToTransfer[i].Hash, err = utils.HashFile(fullPath, c.Options.HashAlgorithm) + log.Debugf("hashed %s to %x using %s", fullPath, c.FilesToTransfer[i].Hash, c.Options.HashAlgorithm) totalFilesSize += fstats.Size() if err != nil { return @@ -1261,6 +1262,7 @@ func (c *Client) updateIfRecipientHasFileInfo() (err error) { } log.Debugf("%s %+x %+x %+v", fileInfo.Name, fileHash, fileInfo.Hash, errHash) if !bytes.Equal(fileHash, fileInfo.Hash) { + log.Debugf("hashed %s to %x using %s", fileInfo.Name, fileHash, c.Options.HashAlgorithm) log.Debugf("hashes are not equal %x != %x", fileHash, fileInfo.Hash) if errHash == nil && !c.Options.Overwrite && errRecipientFile == nil { log.Debug("asking to overwrite") From d6af319ad815a1bc6fdeef6b6299d883fc5cb027 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Tue, 20 Apr 2021 15:48:13 -0700 Subject: [PATCH 6/6] fix tests --- src/croc/croc.go | 3 +++ src/utils/utils_test.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/croc/croc.go b/src/croc/croc.go index 1c41c6d..10ea2b8 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -224,6 +224,9 @@ func (c *Client) sendCollectFiles(options TransferOptions) (err error) { } log.Debugf("%+v", c.FilesToTransfer[i]) } + if c.Options.HashAlgorithm == "" { + c.Options.HashAlgorithm = "xxhash" + } c.FilesToTransfer[i].Hash, err = utils.HashFile(fullPath, c.Options.HashAlgorithm) log.Debugf("hashed %s to %x using %s", fullPath, c.FilesToTransfer[i].Hash, c.Options.HashAlgorithm) totalFilesSize += fstats.Size() diff --git a/src/utils/utils_test.go b/src/utils/utils_test.go index 2847148..a94fd7c 100644 --- a/src/utils/utils_test.go +++ b/src/utils/utils_test.go @@ -175,9 +175,9 @@ func TestHashFile(t *testing.T) { if err := tmpfile.Close(); err != nil { panic(err) } - hashed, err := HashFile(tmpfile.Name()) + hashed, err := HashFile(tmpfile.Name(), "xxhash") assert.Nil(t, err) - assert.Equal(t, "18c9673a4bb8325d323e7f24fda9ae1e", fmt.Sprintf("%x", hashed)) + assert.Equal(t, "e66c561610ad51e2", fmt.Sprintf("%x", hashed)) } func TestPublicIP(t *testing.T) {