if w.f != nil {
		if err := w.closeCurrentFile(); err != nil {
			return err
		}

		// This function is a no-op if opts.Compress is false
		// writes an error in JSON form to stderr, if we cannot
		// compress.
		if err := w.compress(); err != nil {
			msg := fmt.Sprintf("unable to compress log file %v: %v, ignoring and moving on", w.f.Name(), err)
			stdErrEnc.Encode(&log.Entry{
				Level:   ErrorKind,
				Message: msg,

	"bufio"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path"
	"runtime"
	"sync"
	"time"

	"github.com/cosnicolaou/pbzip2"
	"github.com/klauspost/compress/s2"
	"github.com/klauspost/compress/zstd"
	gzip "github.com/klauspost/pgzip"
	"github.com/pierrec/lz4"
)

// Max bzip2 concurrency across calls. 50% of GOMAXPROCS.

// in http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
func signV4TrimAll(input string) string {
	// Compress adjacent spaces (a space is determined by
	// unicode.IsSpace() internally here) to one space and return
	return strings.Join(strings.Fields(input), " ")
}

		"",
	)

	ErrInvalidCompressionIncludesValue = newErrFn(
		"Invalid compression include value",
		"Please check the passed value",
		"Compress extensions/mime-types are delimited by `,`. For eg, MINIO_COMPRESS_MIME_TYPES=\"A,B,C\"",
	)

	ErrInvalidReplicationWorkersValue = newErrFn(
		"Invalid value for replication workers",
		"",
		"MINIO_API_REPLICATION_WORKERS: should be > 0",
	)

// It is mainly used to provide a high-level io.Reader
// access to the ETag computed by a low-level io.Reader:
//
//	content := etag.NewReader(r.Body, nil)
//
//	compressedContent := Compress(content)
//	encryptedContent := Encrypt(compressedContent)
//
//	// Now, we need an io.Reader that can access
//	// the ETag computed over the content.
//	reader := etag.Wrap(encryptedContent, content)

		h:            h,
		shardSize:    shardSize,
		canClose:     &sync.WaitGroup{},
	}
	bw.canClose.Add(1)
	go func() {
		defer bw.canClose.Done()

		totalFileSize := int64(-1) // For compressed objects length will be unknown (represented by length=-1)
		if length != -1 {
			bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.

	// Refer to https://cloud.google.com/storage/docs/transcoding#decompressive_transcoding
	// Need to set `Accept-Encoding` header to `gzip` when issuing a GetObject call, to be able
	// to download the object in compressed state.
	// Calling ReadCompressed with true accomplishes that.
	object := gcs.client.Bucket(gcs.Bucket).Object(gcs.getDest(key)).ReadCompressed(true)

	r, err = object.NewRangeReader(ctx, opts.startOffset, opts.length)

	line, col   int
	Disable     *bool   `yaml:"disable" json:"disable"`
	Batch       *int    `yaml:"batch" json:"batch"`
	InMemory    *bool   `yaml:"inmemory" json:"inmemory"`
	Compress    *bool   `yaml:"compress" json:"compress"`
	SmallerThan *string `yaml:"smallerThan" json:"smallerThan"`
	SkipErrs    *bool   `yaml:"skipErrs" json:"skipErrs"`
}

var _ yaml.Unmarshaler = &BatchJobSnowball{}

//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"bytes"
	"compress/gzip"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"math/rand"
	"net/url"
	"time"

	"github.com/minio/madmin-go/v3"
)