- **Admin Path**: Crawler > Web
- **Supports**: HTTP/HTTPS websites
- **Features**: JavaScript rendering, authentication, custom headers

**File Crawling**:
- **Admin Path**: Crawler > File
- **Supports**: SMB, FTP, local file systems
- **Features**: Access control, file type filtering

**Data Store Crawling**:
- **Admin Path**: Crawler > Data Store
- **Supports**: Databases (via JDBC), CSV files, custom plugins

Anis Eleuch <******@****.***> 1719565609 +0100

Joe Tsai <******@****.***> 1472512232 -0700

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *

package grid

import (
	"context"
	"fmt"
	"net"
	"net/http"
	"net/http/httptest"
	"sync"
	"time"

	"github.com/minio/mux"
)

//go:generate stringer -type=debugMsg $GOFILE

// debugMsg is a debug message for testing purposes.
// may only be used for tests.
type debugMsg int

const (
	debugPrint = false
	debugReqs  = false
)

const (
	debugShutdown debugMsg = iota

	}
	for _, f := range magicHeaders {
		if bytes.Equal(f.header, z[:len(f.header)]) {
			return f.f
		}
	}
	return formatUnknown
}

//go:generate stringer -type=format -trimprefix=format $GOFILE
type format int

const (
	formatUnknown format = iota
	formatGzip
	formatZstd
	formatLZ4
	formatS2
	formatBZ2
)

var magicHeaders = []struct {
	header []byte
	f      format
}{
	{

//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package bandwidth

//go:generate msgp -file=$GOFILE -unexported

import (
	"context"
	"slices"
	"sync"
	"time"

	"golang.org/x/time/rate"
)

//msgp:ignore bucketThrottle Monitor

type bucketThrottle struct {
	*rate.Limiter

//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

//go:generate msgp -file=$GOFILE

package cmd

import (
	"context"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/uuid"
	"github.com/minio/madmin-go/v3"

//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

//go:generate msgp -file=$GOFILE -unexported

package cmd

import (
	"time"

	"github.com/minio/madmin-go/v3"
)

const (
	sizeLessThan1KiB = iota
	sizeLessThan1MiB
	sizeLessThan10MiB
	sizeLessThan100MiB

        }
    }

    /**
     * Handles file download requests for SMB files.
     * @param req the HTTP servlet request
     * @param resp the HTTP servlet response
     * @param file the SMB file to download
     * @throws IOException if an I/O error occurs
     */
    protected void doFile(final HttpServletRequest req, final HttpServletResponse resp, final SmbFile file) throws IOException {