*/
    void addIncludeUrlFilter(String sessionId, String url);

    /**
     * Adds a list of URLs to the include filter for a given session.
     *
     * @param sessionId the ID of the session for which the URLs should be included
     * @param urlList the list of URLs to be added to the include filter
     */
    void addIncludeUrlFilter(String sessionId, List<String> urlList);

    /**

    /** Parameters for the extraction process */
    private final Map<String, String> params;

    /** Container for accessing crawler components */
    private final CrawlerContainer crawlerContainer;

    /** MIME type of the content */
    private String mimeType;

    /** Filename of the content */
    private String filename;

    /** Cache file size threshold in bytes */

                    final Map<String, Object> fields = searchHit.getSourceAsMap();
                    target.setParentUrl(getFieldValue(fields.get("parentUrl"), String.class));
                    target.setMethod(getFieldValue(fields.get("method"), String.class));
                    target.setMimeType(getFieldValue(fields.get("mimeType"), String.class));

     */
    @Override
    public void iterate(final String sessionId, final AccessResultCallback<AccessResultImpl<Long>> accessResultCallback) {
        final Map<String, AccessResultImpl<Long>> arMap = dataHelper.getAccessResultMap(sessionId);
        for (final Map.Entry<String, AccessResultImpl<Long>> entry : arMap.entrySet()) {
            accessResultCallback.iterate(entry.getValue());
        }
    }

    @Resource
    private CrawlerContainer crawlerContainer;

    @Override
    protected String prepareConfigFile() {
        return "crawler.xml";
    }

    public void test_execute_web() throws Exception {
        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

        final String url = "http://localhost:7070/";
        try {
            final int maxCount = 50;

    private final boolean alphabetic;

    private final boolean digit;

    private final boolean letter;

    public CharTypeFilterFactory(final IndexSettings indexSettings, final Environment environment, final String name,
            final Settings settings) {
        super(indexSettings, name, settings);

        alphabetic = settings.getAsBoolean("alphabetic", true);
        digit = settings.getAsBoolean("digit", true);

     * @param in The input stream.
     * @param params The parameters.
     * @return The extracted data.
     */
    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }
        try {
            @SuppressWarnings("resource")

     * Constructs a new {@link FullWidthToHalfWidthAlphabetNormalizer}.
     */
    public FullWidthToHalfWidthAlphabetNormalizer() {
        // nothing
    }

    @Override
    public String normalize(final String text, final String field, final String... langs) {
        final char[] chars = new char[text.length()];
        for (int i = 0; i < chars.length; i++) {
            final char c = text.charAt(i);
            if (c >= 'ａ' && c <= 'ｚ') {

     */
    public void addSitemaps(final String[] sitemaps) {
        sitemapsLocal.set(sitemaps);
    }

    /**
     * Removes sitemaps from the thread-local storage and returns them.
     * @return An array of sitemap URLs, or null if none were present.
     */
    public String[] removeSitemaps() {
        final String[] sitemaps = sitemapsLocal.get();
        if (sitemaps != null) {

    private final boolean step4;

    private final boolean step5;

    private final boolean step6;

    public FlexiblePorterStemFilterFactory(final IndexSettings indexSettings, final Environment environment, final String name,
            final Settings settings) {
        super(indexSettings, name, settings);

        step1 = settings.getAsBoolean("step1", true);
        step2 = settings.getAsBoolean("step2", true);