From 4f8b2aba9bc7ad74d49134483dce5cb16ea0e295 Mon Sep 17 00:00:00 2001 From: VRDighe Date: Tue, 13 Sep 2022 20:08:07 +0530 Subject: [PATCH 1/2] Added zstd as a compression algorithm --- go.mod | 1 + go.sum | 2 ++ src/compress/compress.go | 15 ++++++--------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 9492798..f68a0df 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( ) require ( + github.com/DataDog/zstd v1.5.2 github.com/OneOfOne/xxhash v1.2.5 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index d585298..c04c635 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= diff --git a/src/compress/compress.go b/src/compress/compress.go index 3343822..6964c78 100644 --- a/src/compress/compress.go +++ b/src/compress/compress.go @@ -2,9 +2,10 @@ package compress import ( "bytes" - "compress/flate" "io" + "github.com/DataDog/zstd" + log "github.com/schollz/logger" ) @@ -30,22 +31,18 @@ func Decompress(src []byte) []byte { return deCompressedData.Bytes() } -// compress uses flate to compress a byte slice to a corresponding level +// compress uses zstd to compress a byte slice to a corresponding level func compress(src []byte, dest io.Writer, level int) { - compressor, err := flate.NewWriter(dest, level) - if err != nil { - log.Debugf("error level data: %v", err) - return - } + compressor := zstd.NewWriterLevel(dest, level) if _, err := compressor.Write(src); err != nil { log.Debugf("error writing data: %v", err) } compressor.Close() } -// compress uses flate to decompress an io.Reader +// decompress uses zstd to decompress an io.Reader func decompress(src io.Reader, dest io.Writer) { - decompressor := flate.NewReader(src) + decompressor := zstd.NewReader(src) if _, err := io.Copy(dest, decompressor); err != nil { log.Debugf("error copying data: %v", err) } From e640e4fce0c1a7fb2b1bd0c72dd13b5e9bb2e843 Mon Sep 17 00:00:00 2001 From: VRDighe Date: Fri, 16 Sep 2022 10:19:44 +0530 Subject: [PATCH 2/2] Replaced zstd package --- go.mod | 1 + go.sum | 2 ++ src/compress/compress.go | 14 ++++++++++---- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index f68a0df..e0ee10b 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/denisbrodbeck/machineid v1.0.1 github.com/kalafut/imohash v1.0.2 + github.com/klauspost/compress v1.15.9 github.com/schollz/cli/v2 v2.2.1 github.com/schollz/logger v1.2.0 github.com/schollz/mnemonicode v1.0.1 diff --git a/go.sum b/go.sum index c04c635..1e6bfbe 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbj github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kalafut/imohash v1.0.2 h1:j/cUPa15YvXv7abJlM+kdJIycbBMpmO7WqhPl4YB76I= github.com/kalafut/imohash v1.0.2/go.mod h1:PjHBF0vpo1q7zMqiTn0qwSTQU2wDn5QIe8S8sFQuZS8= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/src/compress/compress.go b/src/compress/compress.go index 6964c78..e78757e 100644 --- a/src/compress/compress.go +++ b/src/compress/compress.go @@ -4,7 +4,7 @@ import ( "bytes" "io" - "github.com/DataDog/zstd" + "github.com/klauspost/compress/zstd" log "github.com/schollz/logger" ) @@ -33,8 +33,11 @@ func Decompress(src []byte) []byte { // compress uses zstd to compress a byte slice to a corresponding level func compress(src []byte, dest io.Writer, level int) { - compressor := zstd.NewWriterLevel(dest, level) - if _, err := compressor.Write(src); err != nil { + compressor, err := zstd.NewWriter(dest, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(level))) + if err != nil { + panic(err) + } + if _, err = compressor.Write(src); err != nil { log.Debugf("error writing data: %v", err) } compressor.Close() @@ -42,7 +45,10 @@ func compress(src []byte, dest io.Writer, level int) { // decompress uses zstd to decompress an io.Reader func decompress(src io.Reader, dest io.Writer) { - decompressor := zstd.NewReader(src) + decompressor, err := zstd.NewReader(src) + if err != nil { + panic(err) + } if _, err := io.Copy(dest, decompressor); err != nil { log.Debugf("error copying data: %v", err) }