Crawler - Code Search

fess-crawler/src/main/java/org/codelibs/fess/crawler/Crawler.java

 *
 * <p>Example usage:
 * <pre>
 *   Crawler crawler = new Crawler();
 *   crawler.addUrl("http://example.com/");
 *   crawler.execute();
 *   crawler.close();
 * </pre>
 */
public class Crawler implements Runnable, AutoCloseable {

    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /**
     * Service for managing URL queues during crawling.
     */

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sun Jul 06 02:13:03 UTC 2025

- 14K bytes

- Viewed (0)

github.com/codelibs/fess

src/main/java/org/codelibs/fess/exec/Crawler.java

 * </pre>
 */
public class Crawler {

    /**
     * Creates a new instance of Crawler.
     */
    public Crawler() {
        // Default constructor
    }

    /** Logger instance for this class. */
    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /** Thread name for web and file system crawling process. */

Registered: Thu Sep 04 12:52:25 UTC 2025

- Last Modified: Thu Aug 07 03:06:29 UTC 2025

- 31K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler-lasta/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java

            crawler1.addUrl(url1);
            crawler1.getCrawlerContext().setMaxAccessCount(maxCount);
            crawler1.getCrawlerContext().setNumOfThread(numOfThread);

            final Crawler crawler2 = crawlerContainer.getComponent("crawler");
            crawler2.setSessionId(crawler2.getSessionId() + "2");
            crawler2.setBackground(true);

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sat Sep 06 04:15:37 UTC 2025

- 12.8K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

README.md

```java
// Create multiple crawler instances
Crawler crawler1 = container.getComponent("crawler");
crawler1.setSessionId("session1");
crawler1.addUrl("https://site1.com");

Crawler crawler2 = container.getComponent("crawler");  
crawler2.setSessionId("session2");
crawler2.addUrl("https://site2.com");

// Execute concurrently
crawler1.setBackground(true);
crawler2.setBackground(true);

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sun Aug 31 05:32:52 UTC 2025

- 15.3K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java

            crawler1.addUrl(url1);
            crawler1.getCrawlerContext().setMaxAccessCount(maxCount);
            crawler1.getCrawlerContext().setNumOfThread(numOfThread);

            final Crawler crawler2 = container.getComponent("crawler");
            crawler2.setSessionId(crawler2.getSessionId() + "2");
            crawler2.setBackground(true);

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sat Sep 06 04:15:37 UTC 2025

- 19.1K bytes

- Viewed (0)

github.com/codelibs/fess

src/test/java/org/codelibs/fess/exec/CrawlerTest.java

        Crawler.Options options = new Crawler.Options();
        options.sessionId = "test-session";
        options.fileConfigIds = "file1";

        int result = crawler.doCrawl(options);
        assertEquals(Constants.EXIT_OK, result);
    }

    public void test_doCrawl_withDataConfigIds() {
        Crawler.Options options = new Crawler.Options();
        options.sessionId = "test-session";

Registered: Thu Sep 04 12:52:25 UTC 2025

- Last Modified: Tue Aug 19 14:09:36 UTC 2025

- 30.5K bytes

- Viewed (0)

github.com/codelibs/fess

src/main/java/org/codelibs/fess/indexer/IndexUpdater.java

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResult;
import org.codelibs.fess.crawler.entity.OpenSearchUrlQueue;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;

Registered: Thu Sep 04 12:52:25 UTC 2025

- Last Modified: Tue Aug 19 14:09:36 UTC 2025

- 32.7K bytes

- Viewed (0)

github.com/codelibs/fess

src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.CrawlerStatus;
import org.codelibs.fess.crawler.interval.FessIntervalController;
import org.codelibs.fess.crawler.service.impl.OpenSearchDataService;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlFilterService;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlQueueService;

Registered: Thu Sep 04 12:52:25 UTC 2025

- Last Modified: Thu Jul 17 08:28:31 UTC 2025

- 24.9K bytes

- Viewed (0)

github.com/codelibs/fess

src/test/resources/plugin/repo2/index.html

            <td>Fri Jan 13 13:53:34 UTC 2017</td>
            <td align="right">
                              &nbsp;
                          </td>
            <td></td>
          </tr>
                  <tr>
            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-crawler-db-mysql/">fess-crawler-db-mysql/</a></td>

Registered: Thu Sep 04 12:52:25 UTC 2025

- Last Modified: Tue Aug 13 07:34:14 UTC 2019

- 10.5K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler/src/main/java/org/codelibs/fess/crawler/processor/impl/DefaultResponseProcessor.java

import org.codelibs.core.lang.SystemUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.entity.UrlQueue;

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Thu Aug 07 02:55:08 UTC 2025

- 12.5K bytes

- Viewed (0)

Search Options