crawler1.addUrl(url1);
            crawler1.getCrawlerContext().setMaxAccessCount(maxCount);
            crawler1.getCrawlerContext().setNumOfThread(numOfThread);

            final Crawler crawler2 = crawlerContainer.getComponent("crawler");
            crawler2.setSessionId(crawler2.getSessionId() + "2");
            crawler2.setBackground(true);

labels.wizard_crawling_config_path=Crawlpad
labels.wizard_button_register_again=Continu aanmaken
labels.wizard_button_register_next=Maken
labels.wizard_start_crawling_title=Start crawlen
labels.wizard_start_crawler_title=Crawler
labels.wizard_start_crawling_desc=U kunt nu beginnen met crawlen door op de knop "Start crawlen" te klikken.
labels.wizard_button_start_crawling=Start crawlen
labels.wizard_button_finish=Overslaan

            crawler1.getCrawlerContext().setMaxAccessCount(maxCount);
            crawler1.getCrawlerContext().setNumOfThread(numOfThread);

            final Crawler crawler2 = getComponent(Crawler.class);
            crawler2.setBackground(true);
            ((UrlFilterImpl) crawler2.urlFilter).setIncludeFilteringPattern("$1$2$3.*");
            crawler2.addUrl(url2);
            crawler2.getCrawlerContext().setMaxAccessCount(maxCount);

            crawler1.addUrl(url1);
            crawler1.getCrawlerContext().setMaxAccessCount(maxCount);
            crawler1.getCrawlerContext().setNumOfThread(numOfThread);

            final Crawler crawler2 = container.getComponent("crawler");
            crawler2.setSessionId(crawler2.getSessionId() + "2");
            crawler2.setBackground(true);

- **Web Framework**: LastaFlute (MVC framework)
- **DI Container**: Lasta Di
- **Data Access**: DBFlute (type-safe ORM for OpenSearch)
- **Search Engine**: OpenSearch
- **App Server**: Embedded Tomcat
- **Crawler**: fess-crawler library
- **Scheduler**: Lasta Job

## Key Directories

```
src/main/java/org/codelibs/fess/
├── FessBoot.java              # Application entry point
├── app/

     * command. Even though search engine crawlers may consider this information
     * when making decisions, they may crawl pages marked "hourly" less
     * frequently than that, and they may crawl pages marked "yearly" more
     * frequently than that. Crawlers may periodically crawl pages marked
     * "never" so that they can handle unexpected changes to those pages.
     */

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.CrawlerStatus;
import org.codelibs.fess.crawler.interval.FessIntervalController;
import org.codelibs.fess.crawler.service.impl.OpenSearchDataService;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlFilterService;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlQueueService;

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

User-agent: Crawler/2.0
Disallow: /ccc

User-agent: Hoge Crawler

        assertFalse(robotsTxt.allows("/aaa", "Crawler"));
        assertTrue(robotsTxt.allows("/bbb", "Crawler"));
        assertTrue(robotsTxt.allows("/ccc", "Crawler"));
        assertTrue(robotsTxt.allows("/ddd", "Crawler"));
        assertTrue(robotsTxt.allows("/aaa", "Crawler/1.0"));
        assertFalse(robotsTxt.allows("/bbb", "Crawler/1.0"));
        assertTrue(robotsTxt.allows("/ccc", "Crawler/1.0"));

  -F "file=@documents.bulk"
```

### 4. Direct Crawler Configuration

Instead of importing documents, configure Fess to crawl your data sources:

**Web Crawling**:
- **Admin Path**: Crawler > Web
- **Supports**: HTTP/HTTPS websites
- **Features**: JavaScript rendering, authentication, custom headers

**File Crawling**:
- **Admin Path**: Crawler > File
- **Supports**: SMB, FTP, local file systems