*/
    protected boolean isHtml(final ResponseData responseData) {
        final String mimeType = responseData.getMimeType();
        if ("text/html".equals(mimeType) || "application/xhtml+xml".equals(mimeType)) {
            return true;
        }
        return false;
    }

    /**
     * Adds a rule for extracting child URLs from HTML tags.
     *

            }
            return false;
        }

        // check it in queue
        for (final UrlQueue<Long> urlInQueue : urlQueueList) {
            if (url.equals(urlInQueue.getUrl())) {
                if (logger.isDebugEnabled()) {
                    logger.debug("URL exists in a queue: {}", url);
                }
                return false;
            }
        }

                    String[] retrieved = crawlerContext.removeSitemaps();
                    if (retrieved != null && retrieved.length == 1 && "http://thread1.com/sitemap.xml".equals(retrieved[0])) {
                        successCount.incrementAndGet();
                    }
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();

     */
    protected void putValue(final ExtractData data, final String key, final Object value) {
        try {
            if (value instanceof String) {
                if ("Subject".equals(key)) {
                    data.putValue(key, getDecodeText(value.toString()));
                } else {
                    data.putValue(key, value.toString());
                }

     * @return True if the language is supported
     */
    public static boolean isSupportedLanguage(final String lang) {
        return StringUtil.isNotBlank(lang) && Stream.of(SUPPORTED_LANGUAGES).anyMatch(lang::equals);
    }

    /**
     * Get analyzer names.
     * @return Analyzer names
     */
    protected Set<String> getAnalyzerNames() {
        final GetSettingsResponse response =

        if (dataClass == null) {
            return super.getData(accessResultData);
        }

        final Map<String, Object> dataMap = XmlUtil.getDataMap(accessResultData);
        if (Map.class.equals(dataClass)) {
            return dataMap;
        }

        try {
            final Object obj = dataClass.getDeclaredConstructor().newInstance();
            BeanUtil.copyMapToBean(dataMap, obj);

     * @param sessionId The new session ID.
     */
    public void setSessionId(final String sessionId) {
        if (StringUtil.isNotBlank(sessionId) && !sessionId.equals(crawlerContext.sessionId)) {
            urlQueueService.updateSessionId(crawlerContext.sessionId, sessionId);
            crawlerContext.sessionId = sessionId;
        }
    }

    /**

                }

                response = client.prepareSearchScroll(scrollId).execute().actionGet(settings.getSearchTimeout());
                if (!scrollId.equals(response.getScrollId())) {
                    SuggestUtil.deleteScrollContext(client, scrollId);
                }
                scrollId = response.getScrollId();
            }
        } finally {

    /**
     * Returns data as a file path of String.
     *
     */
    @Override
    public Object getData(final AccessResultData<?> accessResultData) {
        // check transformer name
        if (!getName().equals(accessResultData.getTransformerName())) {
            throw new CrawlerSystemException(
                    "Transformer is invalid. Use " + accessResultData.getTransformerName() + ". This transformer is " + getName() + ".");

     */
    public StandardCrawlerContainer() {
        initialize();
    }

    @Override
    public <T> T getComponent(final String name) {
        if ("crawlerContainer".equals(name)) {
            return (T) this;
        }

        final ComponentDef<?> componentDef = prototypeMap.get(name);
        if (componentDef != null) {
            return (T) componentDef.get();
        }