@DisplayName("writeWireFormat should write header and trailer")
    void testWriteWireFormat() {
        packet.type = SessionServicePacket.SESSION_MESSAGE;
        packet.trailerLength = 10; // Mock trailer will write 10 bytes

        byte[] dst = new byte[50];
        int totalWritten = packet.writeWireFormat(dst, 0);

        assertEquals(14, totalWritten); // 4 header + 10 trailer

	return r.Header.Get(xhttp.AmzContentSha256) == streamingContentSHA256Trailer &&
		r.Method == http.MethodPut
}

// Verify if the request has AWS Streaming Signature Version '4', with unsigned content and trailer.
func isRequestUnsignedTrailerV4(r *http.Request) bool {
	return r.Header.Get(xhttp.AmzContentSha256) == unsignedPayloadTrailer &&
		r.Method == http.MethodPut
}

// Authorization type.
//

	req, err := http.NewRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, "test-cve-object.txt"), nil)
	c.Assert(err, nil)

	req.Body = io.NopCloser(bytes.NewReader([]byte("foobar!\n")))
	req.Trailer = http.Header{}
	req.Trailer.Set("x-amz-checksum-crc32", "rK0DXg==")

	now := UTCNow()

	req = signer.StreamingUnsignedV4(req, "", 8, now)

	maliciousHeaders := http.Header{

 * </pre>
 */
public class Crawler {

    /**
     * Creates a new instance of Crawler.
     */
    public Crawler() {
        // Default constructor
    }

    /** Logger instance for this class. */
    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /** Thread name for web and file system crawling process. */

        Crawler.Options options = new Crawler.Options();
        options.sessionId = "test-session";
        options.fileConfigIds = "file1";

        int result = crawler.doCrawl(options);
        assertEquals(Constants.EXIT_OK, result);
    }

    public void test_doCrawl_withDataConfigIds() {
        Crawler.Options options = new Crawler.Options();
        options.sessionId = "test-session";

 */
package org.codelibs.fess.crawler;

import java.io.File;

import org.codelibs.core.io.ResourceUtil;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlQueueService;

import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;
import org.codelibs.fess.crawler.service.UrlQueueService;
import org.codelibs.fess.crawler.transformer.impl.FileTransformer;
import org.codelibs.fess.crawler.util.CrawlerWebServer;
import org.codelibs.opensearch.runner.OpenSearchRunner;
import org.dbflute.utflute.lastadi.LastaDiTestCase;

});

Crawler crawler = container.getComponent("crawler");
crawler.addUrl("https://example.com");
crawler.crawlerContext.setMaxAccessCount(500);
String sessionId = crawler.execute();
```

### Background Crawling

```java
// Configure for background execution
crawler.setBackground(true);
String sessionId = crawler.execute();

// Check crawling status

import org.codelibs.core.lang.SystemUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.entity.UrlQueue;

import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.exception.ChildUrlsException;