responseData.setUrl("http://example.com/sitemap.xml");
        File file = ResourceUtil.getResourceAsFile("sitemaps/sitemap1.xml");
        responseData.setResponseBody(file, false);
        return responseData;
    }

    private ResponseData getTestData2_OK() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/sitemap.xml.gz");

        esUrlFilter.setSessionId(sessionId);
        esUrlFilter.setFilterType(INCLUDE);
        esUrlFilter.setUrl(url);
        insert(esUrlFilter, OpType.INDEX);
        includeFilterCache.invalidate(sessionId);
    }

    @Override
    public void addIncludeUrlFilter(final String sessionId, final List<String> urlList) {
        final Set<String> invalidateSet = new HashSet<>();

            final UrlQueueImpl<Long> urlQueue = new UrlQueueImpl<>();
            urlQueue.setSessionId(sessionId);
            urlQueue.setMethod(Constants.GET_METHOD);
            urlQueue.setUrl(url);
            urlQueue.setUrl(url);
            urlQueue.setDepth(0);
            urlQueue.setCreateTime(SystemUtil.currentTimeMillis());
            urlQueueList.add(urlQueue);
        }
    }

    /*
     * (non-Javadoc)

        profile.setId("pro");
        settings.addProfile(profile);
        Repository repo1 = new Repository();
        repo1.setUrl("http://apache.org/");
        repo1.setId("test");
        profile.addRepository(repo1);
        Repository repo2 = new Repository();
        repo2.setUrl("http://apache.org/");
        repo2.setId("test");
        profile.addRepository(repo2);

        assertNotNull(rule);
        assertEquals("fileRule", rule.getRuleId());
    }

    public void test_getRule_sitemaps1() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://www.example.com/sitemap1.xml");
        File file = ResourceUtil.getResourceAsFile("sitemaps/sitemap1.xml");
        responseData.setResponseBody(file, false);
        final Rule rule = ruleManager.getRule(responseData);

    public void test_checkMaxContentLength() {
        AbstractCrawlerClient client = new AbstractCrawlerClient() {
        };
        ResponseData responseData = new ResponseData();
        responseData.setUrl("http://test.com/");

        client.checkMaxContentLength(responseData);

        responseData.setContentLength(-1);
        client.checkMaxContentLength(responseData);

        responseData.setContentLength(1000L);

    public void normalize(Model model, ModelBuildingRequest request) {
        if (model == null) {
            return;
        }

        model.setUrl(normalize(model.getUrl()));

        Scm scm = model.getScm();
        if (scm != null) {
            scm.setUrl(normalize(scm.getUrl()));
            scm.setConnection(normalize(scm.getConnection()));
            scm.setDeveloperConnection(normalize(scm.getDeveloperConnection()));
        }

        responseData.setHttpStatusCode(200);
        responseData.setUrl("http://example.com/");
        responseData.setMimeType("text/html");
        return responseData;
    }

    private ResponseData getTestData2() {
        final ResponseData responseData = new ResponseData();
        responseData.setHttpStatusCode(200);
        responseData.setUrl("https://example.com/");
        responseData.setMimeType("text/html");

        urlQueue.setUrl("http://www.id1.com/");

        urlQueueService.insert(urlQueue);

        final OpenSearchUrlQueue urlQueue2 = new OpenSearchUrlQueue();
        urlQueue2.setCreateTime(System.currentTimeMillis());
        urlQueue2.setDepth(1);
        urlQueue2.setMethod("GET");
        urlQueue2.setSessionId("id2");
        urlQueue2.setUrl("http://www.id2.com/");

        urlList = fessXpathTransformer.convertChildUrlList(urlList);
        assertEquals(2, urlList.size());
        assertEquals("http://www.example.com", urlList.get(0).getUrl());
        assertEquals("http://www.test.com", urlList.get(1).getUrl());

        urlList.clear();
        urlList.add(RequestDataBuilder.newRequestData().get().url("feed://www.example.com").build());