*/
public class CrawlerPooledObjectFactoryTest extends PlainTestCase {

    private CrawlerPooledObjectFactory<TestComponent> factory;
    private StandardCrawlerContainer container;

    @Override
    protected void setUp() throws Exception {
        super.setUp();

        // Reset counters before each test
        TestComponent.resetCounter();
        SingletonTestComponent.resetInstanceCount();

 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {
    private static final String ALL_BOTS = "*";

    /** Map of user agent patterns to their corresponding directives. */
    protected final Map<Pattern, Directive> directiveMap = new LinkedHashMap<>();

    /** List of sitemap URLs found in the robots.txt file. */
    private final List<String> sitemapList = new ArrayList<>();

    /**

    public DataService dataService;

    public UrlQueueService urlQueueService;

    public FileTransformer fileTransformer;

    private StandardCrawlerContainer container;

    @Override
    protected void setUp() throws Exception {
        super.setUp();

        final Map<String, String> featureMap = newHashMap();
        featureMap.put("http://xml.org/sax/features/namespaces", "false");

    private final OffsetAttribute offsetAttr = addAttribute(OffsetAttribute.class);

    private final PositionIncrementAttribute posIncAttr = addAttribute(PositionIncrementAttribute.class);

    protected NGramSynonymTokenizer(final int n, final String delimiters, final boolean expand, final boolean ignoreCase,
            final SynonymLoader synonymLoader) {
        this.n = n;
        this.delimiters = delimiters;

 * Tests all functionality including thread safety and concurrent operations.
 */
public class CrawlerContextTest extends PlainTestCase {

    private CrawlerContext crawlerContext;

    @Override
    protected void setUp() throws Exception {
        super.setUp();
        crawlerContext = new CrawlerContext();
    }

    /**
     * Test implementation of UrlFilter for testing
     */

        @Override
        public ResponseProcessor getResponseProcessor() {
            return responseProcessor;
        }
    }

    private TestRuleManager ruleManager;

    @Override
    protected void setUp() throws Exception {
        super.setUp();
        ruleManager = new TestRuleManager();
    }

    /**
     * Test adding a single rule
     */
    public void test_addRule_single() {

    <sub-class-of type="application/vnd.apple.iwork" />
    <glob pattern="*.numbers"/>
  </mime-type>
  <mime-type type="application/x-tika-iworks-protected">
    <sub-class-of type="application/vnd.apple.iwork" />
    <_comment>Password Protected iWorks File</_comment>
  </mime-type>

  <mime-type type="application/vnd.arastra.swi">
    <glob pattern="*.swi"/>
  </mime-type>