/**
 * Helper class for managing data crawling operations in Fess.
 * This class coordinates the execution of data store crawling processes,
 * managing multiple concurrent crawling threads and handling the indexing
 * of crawled documents into the search engine.
 *
 * <p>The DataIndexHelper supports:</p>
 * <ul>
 *   <li>Concurrent crawling of multiple data configurations</li>

    /**
     * The unique identifier of the crawling information entry being edited.
     * This is a required field for identifying which crawling info to update.
     */
    @Required
    @Size(max = 1000)
    public String id;

    /**
     * The session identifier of the crawling session.
     * This is a required field that identifies the specific crawling session.
     * Maximum length is 20 characters.
     */

/**
 * Exception thrown when an error occurs during data store crawling operations.
 * This exception provides information about the URL where the error occurred
 * and whether the crawling process should be aborted.
 */
public class DataStoreCrawlingException extends CrawlingAccessException {

    private static final long serialVersionUID = 1L;

    /**
     * The URL where the crawling error occurred.
     */
    private final String url;

    }

    /**
     * Stores (inserts or updates) a crawling information record.
     * Sets up the store conditions including creation time if not already set,
     * then performs an insert or update operation with immediate refresh.
     *
     * @param crawlingInfo the crawling information entity to store
     * @throws FessSystemException if the crawling information is null
     */

Onların kurslarını denemek isteyebilirsiniz:

* <a href="https://training.talkpython.fm/fastapi-courses" class="external-link" target="_blank">Talk Python Training</a>

    }

    /**
     * Stores crawling information and parameters for the specified session.
     * Creates a new crawling info record if none exists or if create flag is true.
     * Also stores any accumulated information parameters and clears the info map.
     *
     * @param sessionId the session ID for the crawling information
     * @param create if true, creates a new crawling info regardless of existing records

 * <ul>
 * <li>Web crawling - crawls web sites and web content</li>
 * <li>File system crawling - crawls file systems and documents</li>
 * <li>Data store crawling - crawls databases and other data sources</li>
 * <li>Combined crawling - runs multiple crawling types simultaneously</li>
 * </ul>
 *
 * <p>Command line usage:
 * <pre>
 * java org.codelibs.fess.exec.Crawler [options...]

import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;

import jakarta.annotation.PostConstruct;

/**
 * Helper class for managing crawling configurations.
 * Provides functionality to store, retrieve, and manage different types of crawling configurations
 * including web, file, and data configurations. Supports caching and session-based configuration management.
 */
public class CrawlingConfigHelper {

import org.codelibs.fess.crawler.service.UrlQueueService;

import jakarta.annotation.Resource;

/**
 * The Crawler class is the main class for web crawling. It manages the crawling process,
 * including adding URLs to the queue, filtering URLs, managing crawler threads,
 * and handling the overall crawling lifecycle.
 *
 * <p>It implements the Runnable interface to be executed in a separate thread,

        
        // Configure crawling parameters
        crawler.addUrl("https://example.com");
        crawler.crawlerContext.setMaxAccessCount(100);
        crawler.crawlerContext.setNumOfThread(5);
        crawler.urlFilter.addInclude("https://example.com/.*");
        
        // Execute crawling
        String sessionId = crawler.execute();
        System.out.println("Crawling completed. Session ID: " + sessionId);
    }
}