Joe Tsai <******@****.***> 1442390336 -0700

Joe Tsai <******@****.***> 1442390336 -0700

pax-bad-hdr-large.tar XXXXXXXXXXXXXXXXXXXX...

		{"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"},
		{"todo.txt", "Get animal handling license."},
	}
	for _, file := range files {
		hdr := &tar.Header{
			Name: file.Name,
			Mode: 0600,
			Size: int64(len(file.Body)),
		}
		if err := tw.WriteHeader(hdr); err != nil {
			log.Fatal(err)
		}
		if _, err := tw.Write([]byte(file.Body)); err != nil {
			log.Fatal(err)
		}
	}

			header  *Header
			content []byte
		}
		files := []file{}
		for {
			hdr, err := r.Next()
			if err == io.EOF {
				break
			}
			if err != nil {
				return
			}
			buf := bytes.NewBuffer(nil)
			if _, err := io.Copy(buf, r); err != nil {
				continue
			}
			files = append(files, file{header: hdr, content: buf.Bytes()})
		}

		// If we were unable to read anything out of the archive don't

Joe Tsai <******@****.***> 1443691829 -0700

- Replicate objects between buckets on multiple sites

Upcoming Jobs

- Copy objects from NAS to MinIO
- Copy objects from HDFS to MinIO

## Replication Job
To perform replication via batch jobs, you create a job. The job consists of a job description YAML that describes

- Source location from where the objects must be copied from

				Idx      int
				Header   json.RawMessage
				Metadata json.RawMessage
			}
			versions := make([]version, nVers)
			e = decodeVersions(v, nVers, func(idx int, hdr, meta []byte) error {
				var header xlMetaV2VersionHeaderV2
				if _, e := header.UnmarshalMsg(hdr); e != nil {
					return e
				}
				b, e := header.MarshalJSON()
				if e != nil {
					return e
				}
				var buf bytes.Buffer

Joe Tsai <******@****.***> 1472512232 -0700

const (
	maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5
	maxPartsCount             = 10000
	maxPartSize               = 1024 * 1024 * 1024 * 5
	minPartSize               = 1024 * 1024 * 128 // chosen by us to be optimal for HDDs
)

// optimalPartInfo - calculate the optimal part info for a given
// object size.
//
// NOTE: Assumption here is that for any object to be uploaded to any S3 compatible