fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            assertEquals(1, urlSet.size());
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                assertTrue(url.contains("dir1/test3.txt"));
            }

     */
    public void test_robotsTxtUrlSet() {
        Set<String> urlSet = crawlerContext.getRobotsTxtUrlSet();
        assertNotNull(urlSet);
        assertTrue(urlSet instanceof LruHashSet);
        assertTrue(urlSet.isEmpty());

        // Add URLs to default set
        urlSet.add("http://example.com/robots.txt");
        urlSet.add("http://test.com/robots.txt");

        try {
            fsClient.doGet("file://" + path);
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                if (url.indexOf(".svn") < 0) {

            final int depth, final String encoding) {
        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()
                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.getUrlFilter().match(d.getUrl()))
                .map(d -> {

            robotTxtFile.deleteOnExit();

            // sitemaps.xml
            buf = new StringBuilder();
            buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append('\n');
            buf.append("<urlset ").append("xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">").append('\n');
            buf.append("<url>").append('\n');
            buf.append("<loc>http://localhost:7070/index.html</loc>").append('\n');

            robotTxtFile.deleteOnExit();

            // sitemaps.xml
            buf = new StringBuilder();
            buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append('\n');
            buf.append("<urlset ").append("xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">").append('\n');
            buf.append("<url>").append('\n');
            buf.append("<loc>http://localhost:7070/index.html</loc>").append('\n');

        try {
            if (bis.read(bytes) == -1) {
                return false;
            }

            final String preloadDate = new String(bytes, Constants.UTF_8);
            if (preloadDate.indexOf("<urlset") >= 0 || preloadDate.indexOf("<sitemapindex") >= 0 || preloadDate.startsWith("http://")
                    || preloadDate.startsWith("https://")) {
                // XML Sitemaps
                return true;
            }

        }
        final Set<RequestData> urlSet = new HashSet<>(docList.size());
        for (final Map<String, Object> doc : docList) {
            final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
            if (StringUtil.isNotBlank(url)) {
                urlSet.add(RequestDataBuilder.newRequestData().get().url(url).build());
            }
        }

            return;
        }

        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()
                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.urlFilter.match(d.getUrl()))
                .map(d -> {

l(i.dataTypes[0])||!s["*"]&&l("*")}function Gt(e,t){var n,r,i=ce.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&ce.extend(!0,e,r),e}Xt.href=Et.href,ce.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Et.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(Et.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":zt,text:"text/plain",html:"text/ht...