// content. However, there are many exceptions to this rule.
//
// # Single-part Upload
//
// In case of a basic single-part PUT operation - without server
// side encryption or object compression - the ETag of an object
// is its content MD5.
//
// # Multi-part Upload
//
// The ETag of an object does not correspond to its content MD5
// when the object is uploaded in multiple parts via the S3

OkHttp may add headers that are absent from the original request, including `Content-Length`, `Transfer-Encoding`, `User-Agent`, `Host`, `Connection`, and `Content-Type`. It will add an `Accept-Encoding` header for transparent response compression unless the header is already present. If you’ve got cookies, OkHttp will add a `Cookie` header with them.

        } catch (final Exception e) {
            throw new CrawlingAccessException("Failed to parse " + url, e);
        }
    }

    /**
     * Decodes a similar document hash from its compressed and encoded form.
     * Reverses the encoding process applied by encodeSimilarDocHash.
     *
     * @param hash the encoded hash string to decode

3hlDw==","EcAlgo":1,"EcM":2,"EcN":2,"EcBSize":1048576,"EcIndex":4,"EcDist":[4,1,2,3],"CSumAlgo":1,"PartNums":[1],"PartETags":[""],"PartSizes":[101974],"PartASizes":[176837],"Size":101974,"MTime":1634106631319256439,"MetaSys":{"X-Minio-Internal-compression":"a2xhdXNwb3N0L2NvbXByZXNzL3My","X-Minio-Internal-actual-size":"MTc2ODM3","x-minio-internal-objectlock-legalhold-timestamp":"MjAyMS0xMC0xOVQyMjozNTo0Ni4zNTE4MDU3NTda"},"MetaUsr":{"x-amz-object-lock-mode":"COMPLIANCE","x-amz-object-lock-retain-u...

github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=

package store

import (
	"bytes"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"time"

	"github.com/google/uuid"
	jsoniter "github.com/json-iterator/go"
	"github.com/klauspost/compress/s2"
	"github.com/valyala/bytebufferpool"
)

const (
	defaultLimit = 100000 // Default store limit.
	defaultExt   = ".unknown"
	compressExt  = ".snappy"
)

//   - BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
//   - BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
//   - BytesReturned => Current number of bytes of records payload data returned by S3.
//
// For uncompressed files, BytesScanned and BytesProcessed are equal.

type Key struct {
	Name      string
	Compress  bool
	Extension string
	ItemCount int
}

// String returns the filepath name
func (k Key) String() string {
	keyStr := k.Name
	if k.ItemCount > 1 {
		keyStr = fmt.Sprintf("%d:%s", k.ItemCount, k.Name)
	}
	return keyStr + k.Extension + func() string {
		if k.Compress {
			return compressExt
		}
		return ""
	}()
}

pkg compress/lzw, method (*Reader) Close() error
pkg compress/lzw, method (*Reader) Read([]uint8) (int, error)
pkg compress/lzw, method (*Reader) Reset(io.Reader, Order, int)
pkg compress/lzw, method (*Writer) Close() error
pkg compress/lzw, method (*Writer) Reset(io.Writer, Order, int)
pkg compress/lzw, method (*Writer) Write([]uint8) (int, error)
pkg compress/lzw, type Reader struct

OkHttp Brotli Implementation
============================

This module is an implementation of [Brotli][1] compression.
It enables Brotli support in addition to tranparent Gzip support,
provided Accept-Encoding is not set previously.  Modern web servers
must choose to return Brotli responses.  n.b. It is not used for
sending requests.

```java
OkHttpClient client = new OkHttpClient.Builder()
  .addInterceptor(BrotliInterceptor.INSTANCE)
  .build();