import org.codelibs.fess.crawler.service.UrlQueueService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import jakarta.annotation.Resource;

/**
 * Crawler manages/controls a crawling information.
 *
 * @author shinsuke
 *
 */
public class Crawler implements Runnable, AutoCloseable {

    private static final Logger logger = LoggerFactory.getLogger(Crawler.class);

    @Resource

        Queue<String> queue = getUrls();

        String url;
        int count = 0;
        while((url = queue.poll()) != null && count < max) {

            try {
                Logger.info("crawling: " + url);
                HttpUriRequest request = new HttpGet(url);
                HttpResponse response = httpClient.execute(request);
                if(response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {

                if (logger.isDebugEnabled()) {
                    logger.debug("Canceled urlQueue: {}", urlQueue);
                }
                // cancel crawling
                crawlerContext.decrementAndGetAccessCount();
                final List<UrlQueue<?>> newUrlQueueList = new ArrayList<>();
                newUrlQueueList.add(urlQueue);

errors.design_file_name_is_invalid=The file name is invalid.
errors.design_file_is_unsupported_type=The kind of file is unsupported.
errors.failed_to_create_crawling_config_at_wizard=Failed to create a crawling config.
errors.design_editor_disabled=This feature is disabled.
errors.not_found_on_file_system=Not Found: {0}
errors.could_not_open_on_system=Could not open {0}. <br/>Please check if the file is associated with an application.

errors.design_file_name_is_invalid=The file name is invalid.
errors.design_file_is_unsupported_type=The kind of file is unsupported.
errors.failed_to_create_crawling_config_at_wizard=Failed to create a crawling config.
errors.design_editor_disabled=This feature is disabled.
errors.not_found_on_file_system=Not Found: {0}
errors.could_not_open_on_system=Could not open {0}. <br/>Please check if the file is associated with an application.

                }
                default -> null;
                };
            });
        } catch (final Exception e) {
            logger.warn("Failed to access a crawling config cache: {}", configId, e);
            return null;
        }
    }

    public OptionalThing<String> getPipeline(final String configId) {
        final CrawlingConfig config = getCrawlingConfig(configId);

    }

    @Test
    void searchListTest() {
        logger.info("start searchListTest");
        testReadSearchList();
        testDeleteSearchList();
    }

    /**
     * Methods for a Web Crawling Job
     * */
    private static void createWebConfig() {
        final Map<String, Object> requestBody = new HashMap<>();
        final String urls = "https://www.codelibs.org/" + "\n" + "http://failure.url";

![Administrationsoberfläche](https://fess.codelibs.org/_images/fess_admin_dashboard.png)

Sie können in der Administrationsoberfläche (Web, Datei, Datenspeicher) Crawling-Ziele in den Crawler-Konfigurationsseiten registrieren und den Crawler manuell auf der [Scheduler-Seite](https://fess.codelibs.org/14.17/admin/scheduler-guide.html) starten.

## Migration von einem anderen Suchanbieter

                fileConfigService.store(fConfig);
            }
            return configName;
        } catch (final Exception e) {
            logger.warn("Failed to create crawling config: {}", form.crawlingConfigPath, e);
            throwValidationError(messages -> messages.addErrorsFailedToCreateCrawlingConfigAtWizard(GLOBAL),
                    () -> asHtml(path_AdminWizard_AdminWizardConfigJsp));

page.keymatch.max.fetch.size=1000
page.labeltype.max.fetch.size=1000
page.roletype.max.fetch.size=1000
page.user.max.fetch.size=1000
page.role.max.fetch.size=1000
page.group.max.fetch.size=1000
page.crawling.info.param.max.fetch.size=100
page.crawling.info.max.fetch.size=1000
page.data.config.max.fetch.size=100
page.web.config.max.fetch.size=100
page.file.config.max.fetch.size=100
page.duplicate.host.max.fetch.size=1000