protected Thread parentThread;

    protected ThreadGroup crawlerThreadGroup;

    public Crawler() {
        crawlerContext = new CrawlerContext();
        final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmssSSS", Locale.ENGLISH);
        crawlerContext.sessionId = sdf.format(new Date());
    }

    public void addUrl(final String url) {
        try {
            urlQueueService.add(crawlerContext.sessionId, url);

    public static final String CRAWLING_USER_AGENT_SUFFIX = "; +http://fess.codelibs.org/bot.html)";

    public static final String DOCUMENT_INDEX_SUFFIX_PATTERN = "yyyyMMddHHmmssSSS";

    // fess properties
    public static final String USER_INFO_PROPERTY = "user.info";

    public static final String USER_FAVORITE_PROPERTY = "user.favorite";