// doesn't catch Errors and we cannot depend on the order that our
    // transformations run. (So it is possible that the Error being thrown
    // could prevent our second transformations from running).
    SettableFuture<Integer> exceptionInput = SettableFuture.create();
    ListenableFuture<Integer> exceptionComposedFuture =

    // doesn't catch Errors and we cannot depend on the order that our
    // transformations run. (So it is possible that the Error being thrown
    // could prevent our second transformations from running).
    SettableFuture<Integer> exceptionInput = SettableFuture.create();
    ListenableFuture<Integer> exceptionComposedFuture =

        return lenToRead;
      }
    }
  }

  /**
   * @param input the size of the input source
   * @param offset the first argument to {@link ByteSource#slice}
   * @param length the second argument to {@link ByteSource#slice}
   * @param expectRead the number of bytes we expect to read
   */
  private static void assertCorrectSlice(int input, int offset, long length, int expectRead)
      throws IOException {

        FileNotifyInformation info1 = notifications.get(0);
        assertEquals(FileNotifyInformation.FILE_ACTION_ADDED, info1.getAction());
        assertEquals("test1.txt", info1.getFileName());

        // Verify second notification
        FileNotifyInformation info2 = notifications.get(1);
        assertEquals(FileNotifyInformation.FILE_ACTION_MODIFIED, info2.getAction());
        assertEquals("test2.txt", info2.getFileName());
    }

crawler.crawlerContext.setNumOfThread(10);

// Set maximum crawl depth
crawler.crawlerContext.setMaxDepth(3);

// Set request interval (politeness)
crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second
```

### URL Filtering

```java
// Include patterns
crawler.urlFilter.addInclude("https://example.com/.*");
crawler.urlFilter.addInclude(".*\\.pdf$");

// Exclude patterns  

        when(mockHit.getIndex()).thenReturn("test-index");
        when(mockHit.getId()).thenReturn("doc1");

        // Setup scroll response (empty on second call)
        when(mockClient.prepareSearchScroll("scroll1")).thenReturn(mockScrollBuilder);
        when(mockScrollBuilder.setScroll(any(Scroll.class))).thenReturn(mockScrollBuilder);

myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; $MC_COMMAND ; STATUS=$? ; until [ $STATUS = 0 ] do ATTEMPTS=`expr $ATTEMPTS + 1` ; echo \"Failed attempts: $ATTEMPTS\" ; if [ $ATTEMPTS -gt $LIMIT ]; then exit 1 ; fi ; sleep 2 ; # 1 second intervals between attempts $MC_COMMAND ; STATUS=$? ; done ; set -e ; # reset `e` as active return 0 } # checkBucketExists ($bucket) # Check if the bucket exists, by using the exit code of `mc ls` checkBucketExists() { BUCKET=$1 CMD=$(${MC} ls myminio/$BUCKET...

myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; $MC_COMMAND ; STATUS=$? ; until [ $STATUS = 0 ] do ATTEMPTS=`expr $ATTEMPTS + 1` ; echo \"Failed attempts: $ATTEMPTS\" ; if [ $ATTEMPTS -gt $LIMIT ]; then exit 1 ; fi ; sleep 2 ; # 1 second intervals between attempts $MC_COMMAND ; STATUS=$? ; done ; set -e ; # reset `e` as active return 0 } # checkBucketExists ($bucket) # Check if the bucket exists, by using the exit code of `mc ls` checkBucketExists() { BUCKET=$1 CMD=$(${MC} ls myminio/$BUCKET...

myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; $MC_COMMAND ; STATUS=$? ; until [ $STATUS = 0 ] do ATTEMPTS=`expr $ATTEMPTS + 1` ; echo \"Failed attempts: $ATTEMPTS\" ; if [ $ATTEMPTS -gt $LIMIT ]; then exit 1 ; fi ; sleep 2 ; # 1 second intervals between attempts $MC_COMMAND ; STATUS=$? ; done ; set -e ; # reset `e` as active return 0 } # checkBucketExists ($bucket) # Check if the bucket exists, by using the exit code of `mc ls` checkBucketExists() { BUCKET=$1 CMD=$(${MC} ls myminio/$BUCKET...

myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; $MC_COMMAND ; STATUS=$? ; until [ $STATUS = 0 ] do ATTEMPTS=`expr $ATTEMPTS + 1` ; echo \"Failed attempts: $ATTEMPTS\" ; if [ $ATTEMPTS -gt $LIMIT ]; then exit 1 ; fi ; sleep 2 ; # 1 second intervals between attempts $MC_COMMAND ; STATUS=$? ; done ; set -e ; # reset `e` as active return 0 } # checkBucketExists ($bucket) # Check if the bucket exists, by using the exit code of `mc ls` checkBucketExists() { BUCKET=$1 CMD=$(${MC} ls myminio/$BUCKET...