/**
 * Form class for crawling configuration wizard in the admin interface.
 * Contains validation constraints for creating new crawling configurations.
 */
public class CrawlingConfigForm {

    /**
     * Creates a new form instance.
     */
    public CrawlingConfigForm() {
        // Default constructor
    }

    /**
     * Name of the crawling configuration.
     */
    @Required

labels.createdTime=Created Time
labels.depth=Depth
labels.excludedPaths=Excluded Paths for Crawling
labels.excludedUrls=Excluded URLs for Crawling
labels.excludedDocPaths=Excluded Paths for Searching
labels.excludedDocUrls=Excluded URLs for Searching
labels.hostname=Hostname
labels.id=ID
labels.includedPaths=Included Paths for Crawling
labels.includedUrls=Included URLs for Crawling
labels.includedDocPaths=Included Paths for Searching

- Graceful degradation (e.g., robots.txt parsing continues on errors)

---

## Key Components

### Crawler (`Crawler.java`)

Main orchestrator for crawling operations.

**Key Methods**:
```java
String execute()                // Start crawling, return session ID
void addUrl(String url)         // Add URL to queue
void cleanup(String sessionId)  // Clean up session
void stop()                     // Stop gracefully

    }

    /**
     * Fetches URL queue list for the specified session with configurable ordering strategy.
     * Supports sequential (default) and random ordering based on crawling configuration.
     *
     * @param sessionId the crawling session identifier
     * @return list of URL queue entries for processing
     */
    @Override
    protected List<OpenSearchUrlQueue> fetchUrlQueueList(final String sessionId) {

        return ComponentUtil.getIngestFactory();
    }

    /**
     * Processes a result data object for web/file crawling.
     * Default implementation returns the target unchanged.
     *
     * @param target the result data to process
     * @param responseData the response data from crawling
     * @return the processed result data
     */

            }
        }
    }

    /**
     * Delays the crawling process before processing a URL.
     */
    protected abstract void delayBeforeProcessing();

    /**
     * Delays the crawling process after processing a URL.
     */
    protected abstract void delayAfterProcessing();

    /**
     * Delays the crawling process when there are no URLs in the queue.
     */

        }

        if (cae.isDebugEnabled()) {
            logger.debug("[{}] Crawling Access Exception: url={}", failureUrlId, urlQueue.getUrl(), cae);
        } else if (cae.isInfoEnabled()) {
            logger.info("[{}] {}", failureUrlId, cae.getMessage());
        } else if (cae.isWarnEnabled()) {
            logger.warn("[{}] Crawling Access Exception: url={}", failureUrlId, urlQueue.getUrl(), cae);

    private int currentPageNumber;

    /**
     * Crawling information ID.
     */
    public String id;

    /**
     * Session ID for the crawling session.
     */
    public String sessionId;

    /**
     * Creation time of the crawling information.
     */
    public String createdTime;

    /**
     * Clears all pagination state and crawling information fields.

 *   <li>Handling exceptions that may occur during the crawling process.</li>
 * </ol>
 *
 * <p>
 * The thread also manages the active thread count using {@code crawlerContext.activeThreadCountLock}
 * and provides methods for logging messages using {@link LogHelper}.
 * </p>
 *
 * <p>
 * The crawling process continues until the crawler status is {@link CrawlerStatus#DONE} or the

    @CustomSize(maxKey = "form.admin.max.input.size")
    public String urls;

    /**
     * URL patterns to include during crawling.
     */
    @CustomSize(maxKey = "form.admin.max.input.size")
    public String includedUrls;

    /**
     * URL patterns to exclude during crawling.
     */
    @CustomSize(maxKey = "form.admin.max.input.size")
    public String excludedUrls;

    /**