nofollow - Code Search

src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java

                    }
                    if (content.contains(ROBOTS_TAG_NOFOLLOW)) {
                        nofollow = true;
                    }
                }
                if (noindex && nofollow) {
                    logger.info("META(robots=noindex,nofollow): {}", responseData.getUrl());
                    throw new ChildUrlsException(Collections.emptySet(), "#processMetaRobots");
                }

Registered: Thu Sep 04 12:52:25 UTC 2025

- Last Modified: Thu Aug 07 03:06:29 UTC 2025

- 54.4K bytes

- Viewed (0)

github.com/codelibs/fess

src/test/java/org/codelibs/fess/mylasta/direction/FessPropTest.java

        FessConfig fessConfig = new FessConfig.SimpleImpl() {
            @Override
            public String getCrawlerDocumentHtmlPrunedTags() {
                return "script,div#main,p.image,a[rel=nofollow],div[x-y=a-.:_0]";
            }
        };

        PrunedTag[] tags = fessConfig.getCrawlerDocumentHtmlPrunedTagsAsArray();
        assertTrue(matchesTag(tags[0], "<script></script>"));

Registered: Thu Sep 04 12:52:25 UTC 2025

- Last Modified: Tue Aug 19 14:09:36 UTC 2025

- 13K bytes

- Viewed (0)

github.com/codelibs/fess

src/test/java/org/codelibs/fess/crawler/transformer/FessXpathTransformerTest.java

            };
        };

        final ResponseData responseData = new ResponseData();
        responseData.addMetaData("X-Robots-Tag", "nofollow");

        transformer.processXRobotsTag(responseData, new ResultData());
        assertTrue(responseData.isNoFollow());
    }

    public void test_processMetaRobots_no() throws Exception {
        final String data = "<html><body>foo</body></html>";

Registered: Thu Sep 04 12:52:25 UTC 2025

- Last Modified: Sat Jul 19 23:49:30 UTC 2025

- 41.5K bytes

- Viewed (0)

github.com/codelibs/fess

src/main/resources/fess_config.properties

crawler.document.html.canonical.xpath=//LINK[@rel='canonical'][1]/@href
# HTML tags to prune (remove) during document processing.
crawler.document.html.pruned.tags=noscript,script,style,header,footer,aside,nav,a[rel=nofollow]
# Maximum length of digest extracted from HTML documents.
crawler.document.html.max.digest.length=120
# Default language for HTML documents.
crawler.document.html.default.lang=

Registered: Thu Sep 04 12:52:25 UTC 2025

- Last Modified: Sat Jul 05 14:45:37 UTC 2025

- 54.7K bytes

- Viewed (0)

github.com/codelibs/fess

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

    String CRAWLER_DOCUMENT_HTML_CANONICAL_XPATH = "crawler.document.html.canonical.xpath";

    /** The key of the configuration. e.g. noscript,script,style,header,footer,aside,nav,a[rel=nofollow] */
    String CRAWLER_DOCUMENT_HTML_PRUNED_TAGS = "crawler.document.html.pruned.tags";

    /** The key of the configuration. e.g. 120 */

Registered: Thu Sep 04 12:52:25 UTC 2025

- Last Modified: Thu Jul 17 08:28:31 UTC 2025

- 525.6K bytes

- Viewed (1)

github.com/square/okhttp

okhttp/src/jvmTest/kotlin/okhttp3/DuplexTest.kt

      requestBody.close()
      assertThat(responseBody.readUtf8Line()).isNull()
    }
    body.awaitSuccess()
  }

  /**
   * Duplex calls that have follow-ups are weird. By the time we know there's a follow-up we've
   * already split off another thread to stream the request body. Because we permit at most one
   * exchange at a time we break the request stream out from under that writer.
   */
  @Test

Registered: Fri Sep 05 11:42:10 UTC 2025

- Last Modified: Wed Jun 18 12:28:21 UTC 2025

- 24.8K bytes

- Viewed (0)

github.com/square/okhttp

okhttp/src/commonJvmAndroid/kotlin/okhttp3/TrailersSource.kt

 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package okhttp3

import okio.IOException

/**
 * Returns the trailers that follow an HTTP response, blocking if they aren't ready yet.
 * Implementations of this interface should respond to [Call.cancel] by immediately throwing an
 * [IOException].
 *

Registered: Fri Sep 05 11:42:10 UTC 2025

- Last Modified: Mon Jul 07 18:57:05 UTC 2025

- 1.4K bytes

- Viewed (0)

github.com/codelibs/fess

src/main/java/org/codelibs/fess/app/web/admin/wizard/CrawlingConfigForm.java

    /**
     * Path or URL to be crawled by this configuration.
     */
    @Required
    @Size(max = 1000)
    public String crawlingConfigPath;

    /**
     * Maximum depth for crawling (how many levels deep to follow links).
     */
    @Min(value = 0)
    @Max(value = 2147483647)
    @ValidateTypeFailure
    public Integer depth;

    /**
     * Maximum number of pages/documents to access during crawling.
     */

Registered: Thu Sep 04 12:52:25 UTC 2025

- Last Modified: Thu Jul 17 08:28:31 UTC 2025

- 1.9K bytes

- Viewed (0)

github.com/codelibs/fess

src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java

    protected Map<String, String> filePathMap = new HashMap<>();

    /** The name of this thumbnail generator. */
    protected String name;

    /** Maximum number of redirects to follow. */
    protected int maxRedirectCount = 10;

    /** Availability status of this generator. */
    protected Boolean available = null;

    /**
     * Registers this thumbnail generator with the thumbnail manager.

Registered: Thu Sep 04 12:52:25 UTC 2025

- Last Modified: Thu Aug 07 03:06:29 UTC 2025

- 13.3K bytes

- Viewed (0)

github.com/google/guava

.github/workflows/scorecard.yml

          # - you want to enable the Branch-Protection check on a *public* repository, or
          # - you are installing Scorecard on a *private* repository
          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
          # repo_token: ${{ secrets.SCORECARD_TOKEN }}

          # Public repositories:

Registered: Fri Sep 05 12:43:10 UTC 2025

- Last Modified: Wed Sep 03 19:19:31 UTC 2025

- 2.9K bytes

- Viewed (0)

Search Options