sitemap2.xml http://www.example.com/sitemap1.xml.gz 2004-10-01T18:23:17+00:00 http://www.example.com/sitemap2.xml.gz 2005-01-01...

Shinsuke Sugaya <******@****.***> 1444529815 +0900

Shinsuke Sugaya <******@****.***> 1444529815 +0900

<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   <sitemap>
      <loc>http://www.example.com/sitemap1.xml.gz</loc>
      <lastmod>2004-10-01T18:23:17+00:00</lastmod>
   </sitemap>
   <sitemap>
      <loc>http://www.example.com/sitemap2.xml.gz</loc>
      <lastmod>2005-01-01</lastmod>
   </sitemap>

        assertEquals("http://www.example.com/sitemap1.xml.gz", sitemaps[0].getLoc());

        assertEquals("2005-01-01", sitemaps[1].getLastmod());
        assertEquals("http://www.example.com/sitemap2.xml.gz", sitemaps[1].getLoc());

    }

    public void test_parseXmlSitemapsIndexGz() {
        final InputStream in = ResourceUtil.getResourceAsStream("sitemaps/sitemap2.xml.gz");

    }

    public void test_getRule_sitemaps2() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://www.example.com/sitemap1.xml.gz");
        File file = ResourceUtil.getResourceAsFile("sitemaps/sitemap1.xml.gz");
        responseData.setResponseBody(file, false);
        final Rule rule = ruleManager.getRule(responseData);
        assertNotNull(rule);

        assertContentType("image/gif", null, "hoge.gif");

        assertContentType("application/pdf", "extractor/test.pdf", "hoge.pdf");

        assertContentType("application/gzip", "extractor/gz/test.tar.gz", "hoge.tar.gz");
        assertContentType("application/zip", "extractor/zip/test.zip", "hoge.zip");
        assertContentType("application/x-lharc", "extractor/lha/test.lzh", "hoge.lzh"); // TODO is it correct?

    }

    private ResponseData getTestData2_OK() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/sitemap.xml.gz");
        File file = ResourceUtil.getResourceAsFile("sitemaps/sitemap1.xml.gz");
        responseData.setResponseBody(file, false);
        return responseData;
    }

    private ResponseData getTestData3_OK() {

                    || preloadDate.startsWith("https://")) {
                // XML Sitemaps
                return true;
            }
            // gz
            bis.reset();
            return isValid(new GZIPInputStream(bis), false);
        } catch (final Exception e) {
            if (logger.isDebugEnabled()) {

- RTF, WordPerfect

#### PDFs and Images
- PDF documents (text and metadata extraction)
- Images (JPEG, PNG, GIF, TIFF, BMP)
- Image metadata (EXIF, IPTC, XMP)

#### Archives and Compressed Files
- ZIP, TAR, GZ archives
- LHA compression format
- Nested archive extraction

#### Web and Markup
- HTML, XHTML with XPath support
- XML documents
- JSON and structured data

#### Media Files