import org.codelibs.fess.crawler.client.http.config.CredentialsConfig;
import org.codelibs.fess.crawler.client.http.config.CredentialsConfig.CredentialsType;
import org.codelibs.fess.crawler.client.http.config.WebAuthenticationConfig;
import org.codelibs.fess.crawler.client.http.config.WebAuthenticationConfig.AuthSchemeType;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.client.ftp.FtpClient;
import org.codelibs.fess.crawler.client.http.HcHttpClient;
import org.codelibs.fess.crawler.client.smb.SmbClient;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;

  -F "file=@documents.bulk"
```

### 4. Direct Crawler Configuration

Instead of importing documents, configure Fess to crawl your data sources:

**Web Crawling**:
- **Admin Path**: Crawler > Web
- **Supports**: HTTP/HTTPS websites
- **Features**: JavaScript rendering, authentication, custom headers

**File Crawling**:
- **Admin Path**: Crawler > File
- **Supports**: SMB, FTP, local file systems

        helper.setCrawlerWaitMillis(50);

        // Test when crawler is running
        assertTrue(helper.isCrawlerRunning());
        long start = System.currentTimeMillis();
        helper.checkCrawlerStatus();
        long end = System.currentTimeMillis();
        assertTrue(end - start < 100); // Should return quickly

        // Test when crawler is not running
        helper.setCrawlerRunning(false);

<project name="deps" basedir=".">
	<property name="target.dir" value="${basedir}/target/deps" />
	<property name="webinf.dir" value="${basedir}/src/main/webapp/WEB-INF" />
	<property name="crawler.dir" value="${basedir}/src/main/webapp/WEB-INF/env/crawler" />
	<property name="suggest.dir" value="${basedir}/src/main/webapp/WEB-INF/env/suggest" />
	<property name="thumbnail.dir" value="${basedir}/src/main/webapp/WEB-INF/env/thumbnail" />

                case "crawler.hotthread.interval":
                    return "500ms";
                case "crawler.hotthread.snapshots":
                    return "10";
                case "crawler.hotthread.threads":
                    return "3";
                case "crawler.hotthread.timeout":
                    return "30s";
                case "crawler.hotthread.type":
                    return "cpu";

import org.codelibs.fess.app.service.RequestHeaderService;
import org.codelibs.fess.app.service.WebAuthenticationService;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.client.http.HcHttpClient;
import org.codelibs.fess.crawler.client.http.config.WebAuthenticationConfig;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;

     */
    @Execute
    @Secured({ ROLE, ROLE + VIEW })
    public HtmlResponse index(final SearchForm form) {
        return asListHtml();
    }

    /**
     * Displays a paginated list of web crawler configurations.
     *
     * @param pageNumber the page number to display (optional)
     * @param form the search form containing filter criteria
     * @return HTML response with the web config list
     */

    public void setVersionNo(final Long version) {
        asDocMeta().version(version);
    }

    @Override
    public String toString() {
        return "ScheduledJob [available=" + available + ", crawler=" + crawler + ", createdBy=" + createdBy + ", createdTime=" + createdTime
                + ", cronExpression=" + cronExpression + ", jobLogging=" + jobLogging + ", name=" + name + ", scriptData=" + scriptData

        assertTrue(AdminLogAction.isLogFilename("crawler.log"));
        assertTrue(AdminLogAction.isLogFilename("audit.log"));
        assertTrue(AdminLogAction.isLogFilename("application-2024-01-01.log"));
    }

    @Test
    public void test_isLogFilename_withLogGzExtension() {
        assertTrue(AdminLogAction.isLogFilename("fess.log.gz"));
        assertTrue(AdminLogAction.isLogFilename("crawler.log.gz"));