```

### Helpers

**RobotsTxtHelper**: RFC 9309 parsing, user-agent matching, crawl-delay, sitemaps
**SitemapsHelper**: Sitemap XML parsing, index handling
**MimeTypeHelper**: MIME detection via Tika
**EncodingHelper**: Charset detection with BOM
**UrlConvertHelper**: URL normalization

---

## Development Workflow

### Build Commands

```bash
mvn clean install              # Build all

 *   <li>Old document deletion after successful crawling</li>
 *   <li>Crawling execution monitoring and timing</li>
 * </ul>
 */
public class DataIndexHelper {

    /** Logger instance for this class */
    private static final Logger logger = LogManager.getLogger(DataIndexHelper.class);

    /** Parameter key for controlling deletion of old documents */
    private static final String DELETE_OLD_DOCS = "delete_old_docs";

 * Provides user management through command-line tool execution for password changes and user deletion.
 */
public class CommandChain implements AuthenticationChain {

    private static final Logger logger = LogManager.getLogger(CommandChain.class);

    /** Working directory for command execution. */
    protected File workingDirectory = null;

    /** Maximum number of output lines to capture. */

     * Uses scroll and bulk delete operations for efficient deletion of large result sets.
     *
     * @param callback The callback to configure the search request for identifying documents to delete.
     * @throws OpenSearchAccessException if the deletion fails.
     */
    public void delete(final Consumer<SearchRequestBuilder> callback) {

    /** Unique session identifier for this job execution */
    protected String sessionId;

    /** Flag indicating whether to use local Fesen instance */
    protected boolean useLocalFesen = true;

    /** Path to the log file for this job execution */
    protected String logFilePath;

    /** Log level for this job execution */
    protected String logLevel;

				<version>0.8.13</version>
				<executions>
					<execution>
						<goals>
							<goal>prepare-agent</goal>
						</goals>
					</execution>
					<execution>
						<id>report</id>
						<phase>prepare-package</phase>
						<goals>
							<goal>report</goal>
						</goals>
					</execution>
				</executions>
			</plugin>
			<plugin>
				<groupId>com.mycila</groupId>

        final UrlQueue<?> urlQueue = new UrlQueueImpl<>();
        urlQueue.setUrl("http://example.com/");
        urlQueue.setDepth(1);

        when(urlFilter.match(anyString())).thenReturn(true);

        // Use reflection to access protected method
        final java.lang.reflect.Method method = CrawlerThread.class.getDeclaredMethod("isValid", UrlQueue.class);
        method.setAccessible(true);

				<version>0.8.13</version>
				<executions>
					<execution>
						<goals>
							<goal>prepare-agent</goal>
						</goals>
					</execution>
					<execution>
						<id>report</id>
						<phase>prepare-package</phase>
						<goals>
							<goal>report</goal>
						</goals>
					</execution>
				</executions>
			</plugin>
			<plugin>
				<groupId>com.mycila</groupId>

     * This method first refreshes the index, then performs the deletion, and logs the result.
     *
     * @param searchEngineClient the search engine client to use for deletion
     * @param index the index name to delete documents from
     * @param queryBuilder the query to match documents for deletion
     * @return the number of documents that were deleted
     */

 *   <li>Normalize the filename to handle special characters.</li>
 *   <li>Configure whether to use the filename for MIME type detection.</li>
 *   <li>Configure whether to use the filename for MIME type detection when the stream is octet-stream.</li>
 * </ul>
 *
 * <p>
 * The MIME type detection is based on the {@code tika-mimetypes.xml} resource, which is loaded during initialization.
 * </p>
 *
 * <p>
 * Usage:
 * </p>