"timeToResponseInNS": "13309747"
  },
  "remotehost": "127.0.0.1",
  "requestID": "17CDC1F4D7E69123",
  "userAgent": "MinIO (linux; amd64) minio-go/v7.0.70 mc/RELEASE.2024-04-30T17-44-48Z",
  "requestPath": "/testbucket/hosts",
  "requestHost": "localhost:9000",
  "requestHeader": {
    "Accept-Encoding": "zstd,gzip",

     */
    public Set<RequestData> getChildUrlSet() {
        return childUrlSet;
    }

    /**
     * Creates a RequestData object from this response's URL and method.
     *
     * @return a new RequestData object with the URL and method from this response
     */
    public RequestData getRequestData() {
        final RequestData requestData = new RequestData();
        requestData.setMethod(method);

     */
    protected List<RequestData> convertChildUrlList(final List<RequestData> requestDataList) {
        try {
            final UrlConvertHelper urlConvertHelper = crawlerContainer.getComponent("urlConvertHelper");
            for (final RequestData requestData : requestDataList) {
                requestData.setUrl(urlConvertHelper.convert(requestData.getUrl()));
            }
            return requestDataList;

        // Test building empty RequestData
        RequestData data = RequestDataBuilder.newRequestData().build();

        assertNotNull(data);
        assertNull(data.getMethod());
        assertNull(data.getUrl());
        assertEquals(1.0f, data.getWeight(), 0.001f);
    }

    public void test_methodWithStringGet() {
        // Test method(String) with GET

     */
    protected void processProcessChildUrlsByException(final Object... objs) {
        // CrawlerContext crawlerContext = (CrawlerContext) objs[0];
        final UrlQueue<?> urlQueue = (UrlQueue<?>) objs[1];
        @SuppressWarnings("unchecked")
        final Set<RequestData> requestDataSet = (Set<RequestData>) objs[2];
        if (logger.isDebugEnabled()) {

        data.setUrl("https://example.com/test");
        data.setMethod("POST");

        RequestData requestData = data.getRequestData();
        assertNotNull(requestData);
        assertEquals("https://example.com/test", requestData.getUrl());
        assertEquals(Method.POST, requestData.getMethod());
    }

    public void test_closeWithNonTemporaryFile() throws IOException {

            fail("Should throw ChildUrlsException");
        } catch (ChildUrlsException e) {
            Set<RequestData> childUrls = e.getChildUrlList();
            RequestData requestData = childUrls.iterator().next();

            // Verify it's using GET method
            assertEquals(RequestData.Method.GET, requestData.getMethod());
        }
    }

    public void test_duplicateUrlsInSitemap() {

            fsClient.doGet("file://" + path);
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                if (url.indexOf(".svn") < 0) {

        String requestMethod;

        @Override
        public void onRequestStart(final FaultTolerantClient client, final RequestData request) {
            startCount++;
        }

        @Override
        public void onRequest(final FaultTolerantClient client, final RequestData request, final int count) {
            requestCount++;
            requestUrl = request.getUrl();

            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            assertEquals(1, urlSet.size());
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                assertTrue(url.contains("dir1/test3.txt"));
            }
        } finally {