Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 7 of 7 for getChildUrlSet (0.18 sec)

  1. fess-crawler/src/test/java/org/codelibs/fess/crawler/transformer/impl/HtmlTransformerTest.java

            final ResultData resultData = htmlTransformer.transform(responseData);
            assertEquals(content, new String(resultData.getData()));
            assertEquals(1, resultData.getChildUrlSet().size());
            assertEquals("http://hoge/test2.html", resultData.getChildUrlSet().iterator().next().getUrl());
        }
    
        public void test_transform_urllink() {
            String content = "<a href=\"http://fuga/test.html\">test</a>";
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 13.8K bytes
    - Viewed (0)
  2. fess-crawler/src/main/java/org/codelibs/fess/crawler/processor/impl/DefaultResponseProcessor.java

                    }
                    final int depth = urlQueue.getDepth() == null ? 1 : urlQueue.getDepth() + 1;
                    if (crawlerContext.getMaxDepth() < 0 || depth <= crawlerContext.getMaxDepth()) {
                        // add and filter urls
                        storeChildUrls(crawlerContext, resultData.getChildUrlSet(), urlQueue.getUrl(), depth, resultData.getEncoding());
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:47:32 UTC 2024
    - 9K bytes
    - Viewed (0)
  3. fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/ResultData.java

            return transformerName;
        }
    
        public void setTransformerName(final String transformerName) {
            this.transformerName = transformerName;
        }
    
        public Set<RequestData> getChildUrlSet() {
            return childUrlSet;
        }
    
        public void setChildUrlSet(final Set<RequestData> childUrlSet) {
            this.childUrlSet = childUrlSet;
        }
    
        public String getEncoding() {
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 2.2K bytes
    - Viewed (0)
  4. src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java

                                    fessConfig.getIndexFieldClickCount(), fessConfig.getIndexFieldFavoriteCount() });
                    if (document == null) {
                        storeChildUrlsToQueue(urlQueue, getChildUrlSet(searchEngineClient, id));
                        return true;
                    }
    
                    final Date expires = DocumentUtil.getValue(document, fessConfig.getIndexFieldExpires(), Date.class);
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Thu Feb 22 01:37:57 UTC 2024
    - 14.6K bytes
    - Viewed (0)
  5. fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/ResponseData.java

            childUrlSet.add(url);
        }
    
        public void removeChildUrl(final RequestData url) {
            childUrlSet.remove(url);
        }
    
        public Set<RequestData> getChildUrlSet() {
            return childUrlSet;
        }
    
        public RequestData getRequestData() {
            final RequestData requestData = new RequestData();
            requestData.setMethod(method);
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Sat Oct 12 01:40:57 UTC 2024
    - 5.8K bytes
    - Viewed (0)
  6. src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java

                        logger.info("META(robots=noindex): {}", responseData.getUrl());
                        storeChildUrls(responseData, resultData);
                        throw new ChildUrlsException(resultData.getChildUrlSet(), "#processMetaRobots");
                    }
                    if (nofollow) {
                        logger.info("META(robots=nofollow): {}", responseData.getUrl());
                        responseData.setNoFollow(true);
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Thu Oct 24 13:01:38 UTC 2024
    - 42.9K bytes
    - Viewed (0)
  7. fess-crawler/src/main/java/org/codelibs/fess/crawler/transformer/impl/HtmlTransformer.java

                    }
                    requestDataList = convertChildUrlList(requestDataList);
                    resultData.addAllUrl(requestDataList);
                });
    
                resultData.addAllUrl(responseData.getChildUrlSet());
    
                final RequestData requestData = responseData.getRequestData();
                resultData.removeUrl(requestData);
                resultData.removeUrl(getDuplicateUrl(requestData));
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Oct 24 12:16:00 UTC 2024
    - 19.4K bytes
    - Viewed (0)
Back to top