return false;
            };
        };

        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/");
        responseData.addMetaData("X-Robots-Tag", "noindex,nofollow");

        try {
            transformer.processXRobotsTag(responseData, new ResultData());
            fail();
        } catch (ChildUrlsException e) {

            buf.append(' ').append(fileName);
        }
        final String bodyBase = buf.toString().trim();
        responseData.addMetaData(Extractor.class.getSimpleName(), extractor);
        final String body = documentHelper.getContent(crawlingConfig, responseData, bodyBase, dataMap);
        putResultDataBody(dataMap, fessConfig.getIndexFieldContent(), body);