super.setUp();
        final StandardCrawlerContainer container = new StandardCrawlerContainer().singleton("filenameExtractor", FilenameExtractor.class);
        filenameExtractor = container.getComponent("filenameExtractor");
    }

    /**
     * Test extraction with valid filename in parameters.
     */
    public void test_getText_withValidFilename() {

    /**
     * Constructs a new StandardCrawlerContainer and initializes it.
     */
    public StandardCrawlerContainer() {
        initialize();
    }

    @Override
    public <T> T getComponent(final String name) {
        if ("crawlerContainer".equals(name)) {
            return (T) this;
        }

        final ComponentDef<?> componentDef = prototypeMap.get(name);
        if (componentDef != null) {

    protected void setUp() throws Exception {
        super.setUp();
        final StandardCrawlerContainer container = new StandardCrawlerContainer().singleton("csvExtractor", CsvExtractor.class);
        csvExtractor = container.getComponent("csvExtractor");
    }

    public void test_getText() {
        final InputStream in = ResourceUtil.getResourceAsStream("extractor/csv/test.csv");
        final ExtractData extractData = csvExtractor.getText(in, null);

     */
    @Override
    public AccessResultData<String> getAccessResultData() {
        if (!initializedData) {
            final OpenSearchDataService dataService = SingletonLaContainer.getComponent(OpenSearchDataService.class);
            final OpenSearchAccessResult accessResult = dataService.getAccessResult(getSessionId(), getUrl());
            if (accessResult != null && accessResult.accessResultData != null) {

    }

    public WebConfig getWebConfig() {
        if (webConfig == null) {
            final WebConfigService webConfigService = ComponentUtil.getComponent(WebConfigService.class);
            try {
                webConfig = webConfigService.getWebConfig(getWebConfigId()).get();
            } catch (final Exception e) {

     * Attempts to register itself with a "crawlerClientCreator" component if available.
     */
    @PostConstruct
    public void init() {
        try {
            final CrawlerClientCreator creator = crawlerContainer.getComponent("crawlerClientCreator");
            if (creator != null) {
                creator.register(this);
            }
        } catch (final Exception e) {
            if (logger.isDebugEnabled()) {

        super.setUp();
        final StandardCrawlerContainer container = new StandardCrawlerContainer().singleton("textExtractor", TextExtractor.class);
        textExtractor = container.getComponent("textExtractor");
    }

    /**
     * Test successful text extraction with default encoding (UTF-8).
     */
    public void test_getText_withDefaultEncoding() {

        }

        final Object redirectUrlObj = responseData.getMetaDataMap().get(LOCATION_HEADER);
        if (redirectUrlObj instanceof String) {
            final UrlConvertHelper urlConvertHelper = crawlerContainer.getComponent("urlConvertHelper");
            final String redirectUrl;
            if (urlConvertHelper != null) {
                redirectUrl = urlConvertHelper.convert(redirectUrlObj.toString());
            } else {

                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.urlFilter.match(d.getUrl()))
                .map(d -> {
                    final UrlQueue<?> uq = crawlerContainer.getComponent("urlQueue");
                    uq.setCreateTime(SystemUtil.currentTimeMillis());
                    uq.setDepth(depth);
                    uq.setMethod(Constants.GET_METHOD);
                    uq.setParentUrl(url);

        final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
        final long startTime = systemHelper.getCurrentTimeAsLong();

        final IndexUpdateCallback indexUpdateCallback = ComponentUtil.getComponent(IndexUpdateCallback.class);

        final List<String> sessionIdList = new ArrayList<>();
        dataCrawlingThreadList.clear();
        final List<String> dataCrawlingThreadStatusList = new ArrayList<>();