* supporting pattern matching for user agents, path-based access control, and crawl delay settings.</p>
 *
 * <p>Key features:</p>
 * <ul>
 *   <li>Supports multiple user-agent directives with pattern matching</li>
 *   <li>Handles Allow and Disallow rules for path-based access control</li>
 *   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *

        }
        matcher.appendTail(sb);
        return sb.toString().replaceAll("\\s+", " ").trim();
    }

    /**
     * Returns the current encoding setting.
     * @return The current encoding.
     */
    public String getEncoding() {
        return encoding;
    }

    /**
     * Sets the encoding for content extraction.

        }).build(newConfigs().clusterName(clusterName).numOfNode(1));

        // wait for yellow status
        runner.ensureYellow();

        System.setProperty(FesenClient.HTTP_ADDRESS, "localhost:" + runner.node().settings().get("http.port", "9201"));

        super.setUp();
    }

    @Override
    public void tearDown() throws Exception {
        super.tearDown();
        // close runner
        runner.close();

            final ResponseData responseData = ftpClient.doHead("ftp://localhost:" + FTP_PORT + "/text1.txt");
            assertNotNull(responseData.getLastModified());
            assertTrue(responseData.getLastModified().getTime() < new Date().getTime());
            assertNull(responseData.getResponseBody());
        } finally {
            if (server != null) {
                server.stop();
            }
        }
    }

        factory = new CrawlerPooledObjectFactory<>();
        factory.crawlerContainer = container;
    }

    /**
     * Test component class for testing factory operations
     */
    public static class TestComponent {
        private static AtomicInteger instanceCounter = new AtomicInteger(0);
        private final int id;
        private boolean destroyed = false;

        }).build(newConfigs().clusterName(clusterName).numOfNode(1));

        // wait for yellow status
        runner.ensureYellow();

        System.setProperty(FesenClient.HTTP_ADDRESS, "localhost:" + runner.node().settings().get("http.port", "9201"));

        super.setUp();
    }

    @Override
    public void tearDown() throws Exception {
        super.tearDown();
        // close runner
        runner.close();

                if (numberOfReplicas > 0) {
                    source = "{\"settings\":{\"index\":{\"number_of_shards\":" + numberOfShards
                            + ",\"number_of_replicas\":0,\"auto_expand_replicas\":\"0-" + numberOfReplicas + "\"}}}";
                } else {
                    source = "{\"settings\":{\"index\":{\"number_of_shards\":" + numberOfShards + ",\"number_of_replicas\":"

 * Test class for AbstractRule.
 * Tests the abstract rule implementation and its common functionality.
 */
public class AbstractRuleTest extends PlainTestCase {

    /**
     * Concrete implementation of AbstractRule for testing
     */
    public static class TestAbstractRule extends AbstractRule {
        private static final long serialVersionUID = 1L;

        private boolean matchResult = true;
        private int matchCallCount = 0;

 * such as index, size, tags, roles, fields, languages, and exclusion words.
 *
 * <p>Key functionalities include:</p>
 * <ul>
 *   <li>Setting the target index for the search.</li>
 *   <li>Limiting the number of results (size).</li>
 *   <li>Filtering by tags, roles, fields, and languages.</li>
 *   <li>Excluding specific words from the results.</li>

        }).build(newConfigs().clusterName(clusterName).numOfNode(1));

        // wait for yellow status
        runner.ensureYellow();

        System.setProperty(FesenClient.HTTP_ADDRESS, "localhost:" + runner.node().settings().get("http.port", "9201"));

        super.setUp();

        // logging
        System.setProperty("java.util.logging.SimpleFormatter.format", "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %4$s %2$s %5$s%6$s%n");