}
                final int depth = urlQueue.getDepth() == null ? 1 : urlQueue.getDepth() + 1;
                if (crawlerContext.getMaxDepth() < 0 || depth <= crawlerContext.getMaxDepth()) {
                    // add and filter urls
                    storeChildUrls(crawlerContext, resultData.getChildUrlSet(), urlQueue.getUrl(), depth, resultData.getEncoding());
                }

    permissions:
      contents: read
      issues: write
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v5
        with:
          fetch-depth: 0
      - name: Set up JDK 17
        uses: actions/setup-java@v5
        with:
          distribution: 'temurin'
          java-version: '17'
      - name: Install Groovy
        run: sudo apt-get install groovy

         * @param url the URL to crawl
         * @param depth the depth of this URL in the crawling hierarchy
         */
        CrawlRequest(final String url, final int depth) {
            this.url = url;
            this.depth = depth;
        }

        /**
         * Gets the URL of this crawl request.
         *
         * @return the URL to be crawled
         */
        public String getUrl() {

     * @param childUrlList The set of child URLs to store.
     * @param url The parent URL.
     * @param depth The depth of the child URLs.
     */
    protected void storeChildUrls(final Set<RequestData> childUrlList, final String url, final int depth) {
        if (crawlerContext.getMaxDepth() >= 0 && depth > crawlerContext.getMaxDepth()) {
            return;
        }

        // add url and filter

                    }
                  },
                  service);
        } else if (i % DIRECT_EXECUTIONS_PER_THREAD == DIRECT_EXECUTIONS_PER_THREAD - 1) {
          // When at max depth, record stack trace depth
          lengthChecks.add(
              serializer.submit(
                  new Callable<Integer>() {
                    @Override
                    public Integer call() {

      pull-requests: write
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v5
        with:
          persist-credentials: false
          fetch-depth: 0
      - name: Update jdks.yaml
        uses: gradle/update-jdks-action@main
      - name: Add verification comment
        # https://github.com/gradle/gradle-private/issues/4518
        run: |

     * @param parentUrl the parent URL that referenced this child URL
     * @param weight the weight/priority of the child URL
     * @param depth the crawling depth of the child URL
     */
    @Override
    protected void storeChildUrl(final String childUrl, final String parentUrl, final float weight, final int depth) {
        if (StringUtil.isNotBlank(childUrl)) {

    steps:
      - name: Checkout repository
        uses: actions/checkout@v5
        with:
          ref: devprod/upgrade-to-latest-wrapper
          token: ${{ secrets.GITHUB_TOKEN }}
          fetch-depth: 0
      - name: Setup java
        uses: actions/setup-java@v5
        with:
          distribution: temurin
          java-version: 17

```java
// Set maximum number of URLs to crawl
crawler.crawlerContext.setMaxAccessCount(1000);

// Set number of crawler threads
crawler.crawlerContext.setNumOfThread(10);

// Set maximum crawl depth
crawler.crawlerContext.setMaxDepth(3);

// Set request interval (politeness)
crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second
```

### URL Filtering

```java
// Include patterns

        assertEquals("{labels.createdBy}", FessLabels.LABELS_CREATED_BY);
        assertEquals("{labels.createdTime}", FessLabels.LABELS_CREATED_TIME);
        assertEquals("{labels.depth}", FessLabels.LABELS_DEPTH);
        assertEquals("{labels.excludedPaths}", FessLabels.LABELS_EXCLUDED_PATHS);
        assertEquals("{labels.excludedUrls}", FessLabels.LABELS_EXCLUDED_URLS);