asDocMeta().version(version);
    }

    public void addField(final String key, final Object value) {
        fields.put(key, value);
    }

    public String getLogMessage() {
        return getUrl();
    }

    @Override
    public Map<String, Object> toSource() {
        final Map<String, Object> sourceMap = super.toSource();
        if (fields != null) {
            sourceMap.putAll(fields);
        }

    protected void delayBeforeProcessing() {
        final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
        if (urlQueue == null) {
            return;
        }

        final String url = urlQueue.getUrl();
        if (StringUtil.isBlank(url) || url.startsWith("file:")) {
            // not target
            return;
        }

        try {
            final URI uri = new URI(url);

            throw new RuntimeCIFSException("NTLM handshake failed", e);
        }
        this.handshakeComplete = true;
    }

    @Override
    public URL getURL() {
        return this.connection.getURL();
    }

    @Override
    public int getContentLength() {
        handshake();
        return this.connection.getContentLength();
    }

    @Override

            final String baseHref = getBaseHref(document);
            URI uri;
            try {
                uri = new URI(baseHref == null ? responseData.getUrl() : baseHref);
            } catch (final Exception e) {
                uri = new URI(responseData.getUrl());
            }
            final URI baseUri = uri;
            getChildUrlRules(responseData, resultData).forEach(entry -> {

                return null;
            }

            @Override
            public void setMethod(String method) {
            }

            @Override
            public String getUrl() {
                return url;
            }

            @Override
            public void setUrl(String u) {
            }

            @Override
            public String getMetaData() {

            throw new CrawlingAccessException("Unsupported encoding: " + charsetName, e);
        }
    }

    private String getResourceName(final ResponseData responseData) {
        String name = responseData.getUrl();
        final String enc = responseData.getCharSet();

        if (name == null || enc == null) {
            return null;
        }

        name = name.replaceAll("/+$", "");

            assertNotNull(childUrls);
            assertEquals(1, childUrls.size());

            RequestData requestData = childUrls.iterator().next();
            assertEquals("https://example.com/page1", requestData.getUrl());
        }
    }

    public void test_processMultipleSitemaps() {
        // Setup
        ResponseData responseData = new ResponseData();

        assertEquals(content, new String(resultData.getData()));
        assertEquals(1, resultData.getChildUrlSet().size());
        assertEquals("http://hoge/test2.html", resultData.getChildUrlSet().iterator().next().getUrl());
    }

    public void test_transform_urllink() {
        String content = "<a href=\"http://fuga/test.html\">test</a>";
        final byte[] data = new String(content).getBytes();

    }

    public static RemoteRepository toRemoteRepository(Repository repository) {
        RemoteRepository.Builder builder =
                new RemoteRepository.Builder(repository.getId(), repository.getLayout(), repository.getUrl());
        builder.setSnapshotPolicy(toRepositoryPolicy(repository.getSnapshots()));
        builder.setReleasePolicy(toRepositoryPolicy(repository.getReleases()));
        return builder.build();
    }

        }
        final Extractor extractor = extractorFactory.getExtractor(responseData.getMimeType());
        if (logger.isDebugEnabled()) {
            logger.debug("url={}, extractor={}", responseData.getUrl(), extractor);
        }
        return extractor;
    }