}

    public void test_transform() throws Exception {
        final byte[] data = new String("xyz").getBytes();
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://www.example.com/submit?a=1&b=2");
        responseData.setResponseBody(data);
        responseData.setCharSet("UTF-8");
        setBaseDir();
        final ResultData resultData = fileTransformer.transform(responseData);

        responseData1.setUrl("http://www.example.com/page");
        assertTrue(rule.match(responseData1));

        ResponseData responseData2 = new ResponseData();
        responseData2.setUrl("https://api.example.com/v1/users");
        assertTrue(rule.match(responseData2));

        ResponseData responseData3 = new ResponseData();
        responseData3.setUrl("http://other.com/page");

        responseData.setUrl("http://example.com/sitemap.xml");
        File file = ResourceUtil.getResourceAsFile("sitemaps/sitemap1.xml");
        responseData.setResponseBody(file, false);
        return responseData;
    }

    private ResponseData getTestData2_OK() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/sitemap.xml.gz");

        assertNotNull(rule);
        assertEquals("fileRule", rule.getRuleId());
    }

    public void test_getRule_sitemaps1() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://www.example.com/sitemap1.xml");
        File file = ResourceUtil.getResourceAsFile("sitemaps/sitemap1.xml");
        responseData.setResponseBody(file, false);
        final Rule rule = ruleManager.getRule(responseData);

        responseData.setHttpStatusCode(200);
        responseData.setUrl("http://example.com/");
        responseData.setMimeType("text/html");
        return responseData;
    }

    private ResponseData getTestData2() {
        final ResponseData responseData = new ResponseData();
        responseData.setHttpStatusCode(200);
        responseData.setUrl("https://example.com/");
        responseData.setMimeType("text/html");

    public void test_checkMaxContentLength() {
        AbstractCrawlerClient client = new AbstractCrawlerClient() {
        };
        ResponseData responseData = new ResponseData();
        responseData.setUrl("http://test.com/");

        client.checkMaxContentLength(responseData);

        responseData.setContentLength(-1);
        client.checkMaxContentLength(responseData);

        responseData.setContentLength(1000L);

        ResponseData urlOnlyData = new ResponseData();
        urlOnlyData.setUrl("http://example.com");
        ResultData result2 = testTransformer.transform(urlOnlyData);
        assertNotNull(result2);
        assertNotNull(result2.getData());

        // ResponseData with response body
        ResponseData withBodyData = new ResponseData();
        withBodyData.setUrl("http://example.com");

        final OpenSearchUrlFilter esUrlFilter = new OpenSearchUrlFilter();
        esUrlFilter.setSessionId(sessionId);
        esUrlFilter.setFilterType(INCLUDE);
        esUrlFilter.setUrl(url);
        insert(esUrlFilter, OpType.INDEX);
        includeFilterCache.invalidate(sessionId);
    }

    /**
     * Adds multiple include URL filters for the specified session.
     *

        urlQueue.setUrl("http://www.id1.com/");

        urlQueueService.insert(urlQueue);

        final OpenSearchUrlQueue urlQueue2 = new OpenSearchUrlQueue();
        urlQueue2.setCreateTime(System.currentTimeMillis());
        urlQueue2.setDepth(1);
        urlQueue2.setMethod("GET");
        urlQueue2.setSessionId("id2");
        urlQueue2.setUrl("http://www.id2.com/");

        accessResult1.setParentUrl("http://www.parent.com/");
        accessResult1.setRuleId("htmlRule");
        accessResult1.setSessionId("id1");
        accessResult1.setStatus(200);
        accessResult1.setUrl("http://www.id1.com/");

        dataService.store(accessResult1);

        final OpenSearchAccessResult accessResult2 = dataService.getAccessResult("id1", "http://www.id1.com/");
        assertNotNull(accessResult2);