}
        try (final ResponseData responseData = storageClient.doGet("storage://fess/")) {
            fail();
        } catch (ChildUrlsException e) {
            String[] values = e.getChildUrlList().stream().map(d -> d.getUrl()).sorted().toArray(n -> new String[n]);
            assertEquals(3, values.length);
            assertEquals("storage://fess/dir1/", values[0]);

        }

        try {
            smbClient.doGet(baseUrl);
            fail();
        } catch (final ChildUrlsException e) {
            String[] urls = e.getChildUrlList().stream().map(r -> r.getUrl()).sorted().toArray(String[]::new);
            assertEquals(3, urls.length);
            assertEquals(baseUrl + "dir1/", urls[0]);
            assertEquals(baseUrl + "dir3/", urls[1]);

        this.childUrlList = childUrlList;
    }

    /**
     * Returns the list of child URLs.
     * @return The list of child URLs.
     */
    public Set<RequestData> getChildUrlList() {
        return childUrlList;
    }

        try {
            transformer.processXRobotsTag(responseData, new ResultData());
            fail();
        } catch (ChildUrlsException e) {
            assertTrue(e.getChildUrlList().isEmpty());
        } catch (Exception e) {
            fail();
        }
    }

    public void test_processXRobotsTag_noindex() throws Exception {

            ftpClient.doGet("ftp://localhost:" + FTP_PORT + "/");
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            assertEquals(5, urlSet.size());
            final List<String> urlList = urlSet.stream().map(x -> x.getUrl()).sorted().toList();
            assertEquals("ftp://localhost:10021/dir1", urlList.get(0));

        }
        try {
            fsClient.doGet("file://" + path);
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                if (url.indexOf(".svn") < 0) {

                    crawlerStatsHelper.record(keyObj, StatsAction.CHILD_URLS);
                    if (maxDepth == -1 || crawlRequest.getDepth() < maxDepth) {
                        e.getChildUrlList()
                                .stream() //
                                .filter(data -> !processedUrls.contains(data.getUrl())) //

                    } catch (final ChildUrlsException e) {
                        try {
                            final Set<RequestData> childUrlSet = e.getChildUrlList();
                            log(logHelper, LogType.PROCESS_CHILD_URLS_BY_EXCEPTION, crawlerContext, urlQueue, childUrlSet);
                            // add an url