return false;
            };
        };

        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/");
        responseData.addMetaData("X-Robots-Tag", "noindex,nofollow");

        try {
            transformer.processXRobotsTag(responseData, new ResultData());
            fail();
        } catch (ChildUrlsException e) {

        sb.append(getType());
        if (hasClassifier()) {
            sb.append(':');
            sb.append(getClassifier());
        }
    }

    @Override
    public void addMetadata(ArtifactMetadata metadata) {
        if (metadataMap == null) {
            metadataMap = new HashMap<>();
        }

        ArtifactMetadata m = metadataMap.get(metadata.getKey());
        if (m != null) {

                    snapshot.setTimestamp(matcher.group(2));
                    try {
                        snapshot.setBuildNumber(Integer.parseInt(matcher.group(3)));
                        artifact.addMetadata(new SnapshotArtifactRepositoryMetadata(artifact, snapshot));
                    } catch (NumberFormatException e) {
                        logger.warn("Invalid artifact version " + artifact.getVersion() + ": " + e.getMessage());

            buf.append(' ').append(fileName);
        }
        final String bodyBase = buf.toString().trim();
        responseData.addMetaData(Extractor.class.getSimpleName(), extractor);
        final String body = documentHelper.getContent(crawlingConfig, responseData, bodyBase, dataMap);
        putResultDataBody(dataMap, fessConfig.getIndexFieldContent(), body);

            }
            responseData.setHttpStatusCode(httpStatusCode);
            for (final Header header : response.getAllHeaders()) {
                responseData.addMetaData(header.getName(), header.getValue());
            }
            responseData.setMimeType(contentType);
            final Header contentLengthHeader = response.getFirstHeader("Content-Length");