Robot
/
croc
mirror of https://github.com/schollz/croc.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: add support to respect .gitignore files

This commit is contained in:
PThorpe92 2023-09-22 21:15:27 -04:00
parent f91c7a9948
commit 0af35d7149
No known key found for this signature in database
GPG Key ID: 66DB3FBACBDD05CC
5 changed files with 158 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -7,6 +7,7 @@ require (
github.com/chzyer/readline v1.5.1
github.com/denisbrodbeck/machineid v1.0.1
github.com/kalafut/imohash v1.0.2
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/schollz/cli/v2 v2.2.1
github.com/schollz/logger v1.2.0
github.com/schollz/mnemonicode v1.0.2-0.20190421205639-63fa713ece0d

2
go.sum
View File

@ -41,6 +41,8 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
github.com/schollz/cli/v2 v2.2.1 h1:ou22Mj7ZPjrKz+8k2iDTWaHskEEV5NiAxGrdsCL36VU=
github.com/schollz/cli/v2 v2.2.1/go.mod h1:My6bfphRLZUhZdlFUK8scAxMWHydE7k4s2ed2Dtnn+s=
github.com/schollz/logger v1.2.0 h1:5WXfINRs3lEUTCZ7YXhj0uN+qukjizvITLm3Ca2m0Ho=

8
src/cli/cli.go
View File

@ -44,6 +44,7 @@ func Run() (err error) {
app.UsageText = `Send a file:
croc send file.txt
-git to respect your .gitignore
Send multiple files:
croc send file1.txt file2.txt file3.txt
or
@ -70,6 +71,7 @@ func Run() (err error) {
&cli.StringFlag{Name: "text", Aliases: []string{"t"}, Usage: "send some text"},
&cli.BoolFlag{Name: "no-local", Usage: "disable local relay when sending"},
&cli.BoolFlag{Name: "no-multi", Usage: "disable multiplexing"},
&cli.BoolFlag{Name: "git", Usage: "enable .gitignore respect / don't send ignored files"},
&cli.StringFlag{Name: "ports", Value: "9009,9010,9011,9012,9013", Usage: "ports of the local relay (optional)"},
},
HelpName: "croc send",
@ -198,6 +200,7 @@ func send(c *cli.Context) (err error) {
HashAlgorithm: c.String("hash"),
ThrottleUpload: c.String("throttleUpload"),
ZipFolder: c.Bool("zip"),
GitIgnore: c.Bool("git"),
}
if crocOptions.RelayAddress != models.DEFAULT_RELAY {
crocOptions.RelayAddress6 = ""
@ -243,6 +246,9 @@ func send(c *cli.Context) (err error) {
if !c.IsSet("hash") {
crocOptions.HashAlgorithm = rememberedOptions.HashAlgorithm
}
if !c.IsSet("git") {
crocOptions.GitIgnore = rememberedOptions.GitIgnore
}
}
var fnames []string
@ -281,7 +287,7 @@ func send(c *cli.Context) (err error) {
// generate code phrase
crocOptions.SharedSecret = utils.GetRandomName()
}
minimalFileInfos, emptyFoldersToTransfer, totalNumberFolders, err := croc.GetFilesInfo(fnames, crocOptions.ZipFolder)
minimalFileInfos, emptyFoldersToTransfer, totalNumberFolders, err := croc.GetFilesInfo(fnames, crocOptions.ZipFolder, crocOptions.GitIgnore)
if err != nil {
return
}

125
src/croc/croc.go
View File

@ -20,6 +20,7 @@ import (
"golang.org/x/time/rate"
"github.com/denisbrodbeck/machineid"
ignore "github.com/sabhiram/go-gitignore"
log "github.com/schollz/logger"
"github.com/schollz/pake/v3"
"github.com/schollz/peerdiscovery"
@ -77,6 +78,7 @@ type Options struct {
ThrottleUpload string
ZipFolder bool
TestFlag bool
GitIgnore bool
}
// Client holds the state of the croc transfer
@ -101,6 +103,7 @@ type Client struct {
TotalNumberFolders int
FilesToTransferCurrentNum int
FilesHasFinished map[int]struct{}
TotalFilesIgnored int
// send / receive information of current file
CurrentFile *os.File
@ -149,6 +152,7 @@ type FileInfo struct {
Symlink string `json:"sy,omitempty"`
Mode os.FileMode `json:"md,omitempty"`
TempFile bool `json:"tf,omitempty"`
IsIgnored bool `json:"ig,omitempty"`
}
// RemoteFileRequest requests specific bytes
@ -251,9 +255,51 @@ func isEmptyFolder(folderPath string) (bool, error) {
return false, nil
}
// helper function to walk each subfolder and parses against an ignore file.
// returns a hashmap Key: Absolute filepath, Value: boolean (true=ignore)
func gitWalk(dir string, gitObj *ignore.GitIgnore, files map[string]bool) {
var ignoredDir bool
var current string
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if isChild(current, path) && ignoredDir {
files[path] = true
return nil
}
if info.IsDir() && filepath.Base(path) == filepath.Base(dir) {
ignoredDir = false // Skip applying ignore rules for root directory
return nil
}
if gitObj.MatchesPath(info.Name()) {
files[path] = true
ignoredDir = true
current = path
return nil
} else {
files[path] = false
ignoredDir = false
return nil
}
})
if err != nil {
log.Errorf("filepath error")
}
}
func isChild(parentPath, childPath string) bool {
relPath, err := filepath.Rel(parentPath, childPath)
if err != nil {
return false
}
return !strings.HasPrefix(relPath, "..")
}
// This function retrieves the important file information
// for every file that will be transferred
func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyFolders []FileInfo, totalNumberFolders int, err error) {
func GetFilesInfo(fnames []string, zipfolder bool, ignoreGit bool) (filesInfo []FileInfo, emptyFolders []FileInfo, totalNumberFolders int, err error) {
// fnames: the relative/absolute paths of files/folders that will be transferred
totalNumberFolders = 0
var paths []string
@ -271,7 +317,44 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
paths = append(paths, fname)
}
}
var ignoredPaths = make(map[string]bool)
if ignoreGit {
wd, wdErr := os.Stat(".gitignore")
if wdErr == nil {
gitIgnore, gitErr := ignore.CompileIgnoreFile(wd.Name())
if gitErr == nil {
for _, path := range paths {
abs, absErr := filepath.Abs(path)
if absErr != nil {
err = absErr
return
}
if gitIgnore.MatchesPath(path) {
ignoredPaths[abs] = true
}
}
}
}
for _, path := range paths {
abs, absErr := filepath.Abs(path)
if absErr != nil {
err = absErr
return
}
file, fileErr := os.Stat(path)
if fileErr == nil && file.IsDir() {
_, subErr := os.Stat(filepath.Join(path, ".gitignore"))
if subErr == nil {
gitObj, gitObjErr := ignore.CompileIgnoreFile(filepath.Join(path, ".gitignore"))
if gitObjErr != nil {
err = gitObjErr
return
}
gitWalk(abs, gitObj, ignoredPaths)
}
}
}
}
for _, fpath := range paths {
stat, errStat := os.Lstat(fpath)
@ -286,7 +369,6 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
err = errAbs
return
}
if stat.IsDir() && zipfolder {
if fpath[len(fpath)-1:] != "/" {
fpath += "/"
@ -304,7 +386,8 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
err = errAbs
return
}
filesInfo = append(filesInfo, FileInfo{
fInfo := FileInfo{
Name: stat.Name(),
FolderRemote: "./",
FolderSource: filepath.Dir(absPath),
@ -312,7 +395,12 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
ModTime: stat.ModTime(),
Mode: stat.Mode(),
TempFile: true,
})
IsIgnored: ignoredPaths[absPath],
}
if fInfo.IsIgnored {
continue
}
filesInfo = append(filesInfo, fInfo)
continue
}
@ -325,7 +413,7 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
remoteFolder := strings.TrimPrefix(filepath.Dir(pathName),
filepath.Dir(absPath)+string(os.PathSeparator))
if !info.IsDir() {
filesInfo = append(filesInfo, FileInfo{
fInfo := FileInfo{
Name: info.Name(),
FolderRemote: strings.ReplaceAll(remoteFolder, string(os.PathSeparator), "/") + "/",
FolderSource: filepath.Dir(pathName),
@ -333,10 +421,19 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
ModTime: info.ModTime(),
Mode: info.Mode(),
TempFile: false,
})
IsIgnored: ignoredPaths[pathName],
}
if fInfo.IsIgnored && ignoreGit {
return nil
} else {
filesInfo = append(filesInfo, fInfo)
}
} else {
totalNumberFolders++
if ignoredPaths[pathName] {
return filepath.SkipDir
}
isEmptyFolder, _ := isEmptyFolder(pathName)
totalNumberFolders++
if isEmptyFolder {
emptyFolders = append(emptyFolders, FileInfo{
// Name: info.Name(),
@ -352,7 +449,7 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
}
} else {
filesInfo = append(filesInfo, FileInfo{
fInfo := FileInfo{
Name: stat.Name(),
FolderRemote: "./",
FolderSource: filepath.Dir(absPath),
@ -360,9 +457,14 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
ModTime: stat.ModTime(),
Mode: stat.Mode(),
TempFile: false,
})
IsIgnored: ignoredPaths[absPath],
}
if fInfo.IsIgnored && ignoreGit {
continue
} else {
filesInfo = append(filesInfo, fInfo)
}
}
}
return
}
@ -517,7 +619,6 @@ func (c *Client) Send(filesInfo []FileInfo, emptyFoldersToTransfer []FileInfo, t
c.TotalNumberFolders = totalNumberFolders
c.TotalNumberOfContents = len(filesInfo)
err = c.sendCollectFiles(filesInfo)
if err != nil {
return
}

40
src/croc/croc_test.go
View File

@ -5,6 +5,7 @@ import (
"path"
"path/filepath"
"runtime"
"strings"
"sync"
"testing"
"time"
@ -41,6 +42,7 @@ func TestCrocReadme(t *testing.T) {
DisableLocal: true,
Curve: "siec",
Overwrite: true,
GitIgnore: false,
})
if err != nil {
panic(err)
@ -66,7 +68,7 @@ func TestCrocReadme(t *testing.T) {
var wg sync.WaitGroup
wg.Add(2)
go func() {
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../README.md"}, false)
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../README.md"}, false, false)
if errGet != nil {
t.Errorf("failed to get minimal info: %v", errGet)
}
@ -132,7 +134,7 @@ func TestCrocEmptyFolder(t *testing.T) {
var wg sync.WaitGroup
wg.Add(2)
go func() {
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false)
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)
if errGet != nil {
t.Errorf("failed to get minimal info: %v", errGet)
}
@ -174,6 +176,7 @@ func TestCrocSymlink(t *testing.T) {
DisableLocal: true,
Curve: "siec",
Overwrite: true,
GitIgnore: false,
})
if err != nil {
panic(err)
@ -199,7 +202,7 @@ func TestCrocSymlink(t *testing.T) {
var wg sync.WaitGroup
wg.Add(2)
go func() {
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false)
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)
if errGet != nil {
t.Errorf("failed to get minimal info: %v", errGet)
}
@ -229,6 +232,32 @@ func TestCrocSymlink(t *testing.T) {
t.Errorf("symlink transfer failed: %s", err.Error())
}
}
func testCrocIgnoreGit(t *testing.T) {
log.SetLevel("trace")
defer os.Remove(".gitignore")
time.Sleep(300 * time.Millisecond)
time.Sleep(1 * time.Second)
file, err := os.Create(".gitignore")
if err != nil {
log.Errorf("error creating file")
}
_, err = file.WriteString("LICENSE")
if err != nil {
log.Errorf("error writing to file")
}
time.Sleep(1 * time.Second)
// due to how files are ignored in this function, all we have to do to test is make sure LICENSE doesn't get included in FilesInfo.
filesInfo, _, _, errGet := GetFilesInfo([]string{"../../LICENSE", ".gitignore", "croc.go"}, false, true)
if errGet != nil {
t.Errorf("failed to get minimal info: %v", errGet)
}
for _, file := range filesInfo {
if strings.Contains(file.Name, "LICENSE") {
t.Errorf("test failed, should ignore LICENSE")
}
}
}
func TestCrocLocal(t *testing.T) {
log.SetLevel("trace")
@ -249,6 +278,7 @@ func TestCrocLocal(t *testing.T) {
DisableLocal: false,
Curve: "siec",
Overwrite: true,
GitIgnore: false,
})
if err != nil {
panic(err)
@ -276,7 +306,7 @@ func TestCrocLocal(t *testing.T) {
os.Create("touched")
wg.Add(2)
go func() {
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../LICENSE", "touched"}, false)
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../LICENSE", "touched"}, false, false)
if errGet != nil {
t.Errorf("failed to get minimal info: %v", errGet)
}
@ -329,7 +359,7 @@ func TestCrocError(t *testing.T) {
Curve: "siec",
Overwrite: true,
})
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{tmpfile.Name()}, false)
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{tmpfile.Name()}, false, false)
if errGet != nil {
t.Errorf("failed to get minimal info: %v", errGet)
}