*
     * @param offset the offset to set
     */
    public void setOffset(final long offset) {
        this.offset = offset;
    }

    /**
     * Add RDMA channel information for direct memory access
     *
     * @param remoteKey remote memory key
     * @param address remote memory address
     * @param length length of memory region
     */

     * @param url the URL to process
     * @return a map containing the processed document data
     * @throws CrawlingAccessException if crawling fails or configuration is invalid
     * @throws ChildUrlsException if the URL redirects to another location
     * @throws CrawlerSystemException if data deserialization fails
     */
    public Map<String, Object> processRequest(final CrawlingConfig crawlingConfig, final String crawlingInfoId, final String url) {

    protected final BlockingQueue<RdmaWorkRequest> pendingRequests;

    // Connection state
    /** Current connection state */
    protected volatile RdmaConnectionState state;
    /** Credit management for SMB Direct */
    protected RdmaCredits credits;
    /** Maximum size for fragmented operations */
    protected int maxFragmentedSize;
    /** Maximum size for RDMA read/write operations */
    protected int maxReadWriteSize;

            return in.read(inB);
        }
    }

    @Override
    public int recv(final byte[] buf, final int off, final int len) throws IOException {
        return getInput().readDirect(buf, off, len);

    }

    @Override
    public void send(final byte[] buf, final int off, final int length) throws IOException {
        getOutput().writeDirect(buf, off, length, 1);
    }

crawler.urlFilter.addExclude(".*\\.js$");
crawler.urlFilter.addExclude(".*login.*");
```

## Supported Protocols and Formats

### Protocols
- **HTTP/HTTPS**: Full web crawling support with cookies, authentication, redirects
- **File System**: Local and network file system access
- **FTP**: FTP server crawling with authentication
- **SMB/CIFS**: Windows network shares
- **Storage**: Cloud storage systems (MinIO, S3-compatible)

     */
    public static final File tmpDir = new File(System.getProperty("java.io.tmpdir"));

    /**
     * Protected constructor to prevent direct instantiation but allow inheritance.
     */
    protected Curl() {
        // nothing
    }

    /**
     * Creates a new CurlRequest with the HTTP GET method for the specified URL.
     *

         * the 'received' member of the response object will not
         * be set to true indicating the send and sendTransaction
         * methods that the next part should be sent. This is a
         * very indirect and simple batching control mechanism.
         */

        if (andx == null || !USE_BATCHING || batchLevel >= getBatchLimit(andx.command)) {
            andxCommand = (byte) 0xFF;
            andx = null;

import jcifs.SmbSession;
import jcifs.SmbTransport;
import jcifs.smb.SmbException;
import jcifs.smb.SmbTransportInternal;

/**
 * RDMA-enabled SMB transport that wraps existing SMB transport
 * and adds RDMA capabilities for direct memory access operations.
 */
public class RdmaTransport implements SmbTransportInternal {
    private static final Logger log = LoggerFactory.getLogger(RdmaTransport.class);

    private final SmbTransportInternal delegate;

            off += fragSize;
        }
        throw new IOException();
    }

    /**
     * @param msg
     * @param in
     * @param off
     * @param isDirect
     * @return
     * @throws IOException
     * @throws DcerpcException
     * @throws NdrException
     */

         * the 'received' member of the response object will not
         * be set to true indicating the send and sendTransaction
         * methods that the next part should be sent. This is a
         * very indirect and simple batching control mechanism.
         */

        if (this.andx == null || !getConfig().isUseBatching()
                || this.batchLevel >= getBatchLimit(getConfig(), (byte) this.andx.getCommand())) {