/** The number of a thread */
    protected int numOfThread = 10;

    protected int maxThreadCheckCount = 20;

    /** a max depth for crawling. -1 is no depth check. */
    protected int maxDepth = -1;

    /** a max count to access urls. 0 is no limit to access it. */
    protected long maxAccessCount = 0;

    public String getSessionId() {
        return sessionId;
    }

     * .AccessResultData)
     */
    @Override
    public Object getData(final AccessResultData<?> accessResultData) {
        // check transformer name
        if (!getName().equals(accessResultData.getTransformerName())) {
            throw new CrawlerSystemException(

        } else if (isSuccessful(responseData)) {
            if (transformer == null) {
                if (logger.isDebugEnabled()) {
                    logger.debug("No Transformer for ({}). PLEASE CHECK YOUR CONFIGURATION.", responseData.getUrl());
                }
            } else {
                final ResultData resultData = transformer.transform(responseData);
                if (resultData == null) {

                responseData.setCharSet(charset);
                responseData.setContentLength(0);
            } else if (file.isFile()) {
                // check file size
                responseData.setContentLength(file.length());
                checkMaxContentLength(responseData);

                parseFileOwnerAttribute(responseData, file);

                        response.wait(timeout);

                        /*
                         * JetDirect printer can respond to regular broadcast query
                         * with node status so we need to check to make sure that
                         * the record type matches the question type and if not,
                         * loop around and try again.
                         */

        }

        final List<Map<String, Object>> afterList = readJobLog(NAME_PREFIX);
        assertEquals(0, afterList.size()); // check if logs are successfully deleted
    }

    /**
     * Test for CrawlingInfo
     * */
    private void testReadCrawlingInfo() {
        final List<Map<String, Object>> logList = readCrawlingInfo(webConfigId);

                // exclude app.web.api.admin packages
                        clazz.getName().contains(".app.web.api.admin.")) {
                    return;
                }
                check(srcFile, clazz, webPackageKeyword);
            }
        });
    }

        final FessConfig fessConfig = ComponentUtil.getFessConfig();
        final CrawlerStatsHelper crawlerStatsHelper = ComponentUtil.getCrawlerStatsHelper();
        synchronized (indexUpdateCallback) {
            // required check
            if (!dataMap.containsKey(fessConfig.getIndexFieldUrl()) || dataMap.get(fessConfig.getIndexFieldUrl()) == null) {
                logger.warn("Could not add a doc. Invalid data: {}", dataMap);
                return;

        final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();

        if (logger.isDebugEnabled()) {
            logger.debug("Adding {}", dataMap);
        }

        //   required check
        final Object urlObj = dataMap.get(fessConfig.getIndexFieldUrl());
        if (urlObj == null) {
            throw new DataStoreException("url is null. dataMap=" + dataMap);
        }

    }

    public DynamicProperties(final Path path) {
        this(path == null ? null : path.toFile());
    }

    public DynamicProperties(final File file) {
        // check path
        if (file == null) {
            throw new FileAccessException("ECL0108");
        }

        this.propertiesFile = file;
        if (!this.propertiesFile.exists()) {