crawler.crawlerContext.setNumOfThread(10);

// Set maximum crawl depth
crawler.crawlerContext.setMaxDepth(3);

// Set request interval (politeness)
crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second
```

### URL Filtering

```java
// Include patterns
crawler.urlFilter.addInclude("https://example.com/.*");
crawler.urlFilter.addInclude(".*\\.pdf$");

// Exclude patterns  

        when(mockHit.getIndex()).thenReturn("test-index");
        when(mockHit.getId()).thenReturn("doc1");

        // Setup scroll response (empty on second call)
        when(mockClient.prepareSearchScroll("scroll1")).thenReturn(mockScrollBuilder);
        when(mockScrollBuilder.setScroll(any(Scroll.class))).thenReturn(mockScrollBuilder);

myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; $MC_COMMAND ; STATUS=$? ; until [ $STATUS = 0 ] do ATTEMPTS=`expr $ATTEMPTS + 1` ; echo \"Failed attempts: $ATTEMPTS\" ; if [ $ATTEMPTS -gt $LIMIT ]; then exit 1 ; fi ; sleep 2 ; # 1 second intervals between attempts $MC_COMMAND ; STATUS=$? ; done ; set -e ; # reset `e` as active return 0 } # checkBucketExists ($bucket) # Check if the bucket exists, by using the exit code of `mc ls` checkBucketExists() { BUCKET=$1 CMD=$(${MC} ls myminio/$BUCKET...

myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; $MC_COMMAND ; STATUS=$? ; until [ $STATUS = 0 ] do ATTEMPTS=`expr $ATTEMPTS + 1` ; echo \"Failed attempts: $ATTEMPTS\" ; if [ $ATTEMPTS -gt $LIMIT ]; then exit 1 ; fi ; sleep 2 ; # 1 second intervals between attempts $MC_COMMAND ; STATUS=$? ; done ; set -e ; # reset `e` as active return 0 } # checkBucketExists ($bucket) # Check if the bucket exists, by using the exit code of `mc ls` checkBucketExists() { BUCKET=$1 CMD=$(${MC} ls myminio/$BUCKET...

myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; $MC_COMMAND ; STATUS=$? ; until [ $STATUS = 0 ] do ATTEMPTS=`expr $ATTEMPTS + 1` ; echo \"Failed attempts: $ATTEMPTS\" ; if [ $ATTEMPTS -gt $LIMIT ]; then exit 1 ; fi ; sleep 2 ; # 1 second intervals between attempts $MC_COMMAND ; STATUS=$? ; done ; set -e ; # reset `e` as active return 0 } # checkBucketExists ($bucket) # Check if the bucket exists, by using the exit code of `mc ls` checkBucketExists() { BUCKET=$1 CMD=$(${MC} ls myminio/$BUCKET...

myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; $MC_COMMAND ; STATUS=$? ; until [ $STATUS = 0 ] do ATTEMPTS=`expr $ATTEMPTS + 1` ; echo \"Failed attempts: $ATTEMPTS\" ; if [ $ATTEMPTS -gt $LIMIT ]; then exit 1 ; fi ; sleep 2 ; # 1 second intervals between attempts $MC_COMMAND ; STATUS=$? ; done ; set -e ; # reset `e` as active return 0 } # checkBucketExists ($bucket) # Check if the bucket exists, by using the exit code of `mc ls` checkBucketExists() { BUCKET=$1 CMD=$(${MC} ls myminio/$BUCKET...

    TF_DeleteAbstractOp(add_op);
    // Extract the resulting tensor.
    add_output1 = TF_OutputListGet(add_outputs, 0);
    TF_DeleteOutputList(add_outputs);
  }

  // Same with a second "Add" computing `arg1 + arg1`.
  TF_AbstractTensor* add_output2;
  {
    // Build an abstract operation, inputs and output.
    auto* add_op = TF_NewAbstractOp(graph_ctx);
    TF_AbstractOpSetOpType(add_op, "Add", s);

MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" $MC_COMMAND STATUS=$? until [ $STATUS = 0 ]; do ATTEMPTS=$(expr $ATTEMPTS + 1) echo \"Failed attempts: $ATTEMPTS\" if [ $ATTEMPTS -gt $LIMIT ]; then exit 1 fi sleep 2 # 1 second intervals between attempts $MC_COMMAND STATUS=$? done set -e # reset `e` as active return 0 } # checkBucketExists ($bucket) # Check if the bucket exists, by using the exit code of `mc ls` checkBucketExists() { BUCKET=$1 CMD=$(${MC} stat myminio/$BUCKET...

MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" $MC_COMMAND STATUS=$? until [ $STATUS = 0 ]; do ATTEMPTS=$(expr $ATTEMPTS + 1) echo \"Failed attempts: $ATTEMPTS\" if [ $ATTEMPTS -gt $LIMIT ]; then exit 1 fi sleep 2 # 1 second intervals between attempts $MC_COMMAND STATUS=$? done set -e # reset `e` as active return 0 } # checkBucketExists ($bucket) # Check if the bucket exists, by using the exit code of `mc ls` checkBucketExists() { BUCKET=$1 CMD=$(${MC} stat myminio/$BUCKET...

        dataMap1.put("url", "http://example.com/test1");
        indexUpdateCallback.store(paramMap, dataMap1);
        assertEquals(1, indexingHelper.sendDocumentsCalled);

        // Add second document - should trigger indexing again
        Map<String, Object> dataMap2 = new HashMap<>();
        dataMap2.put("url", "http://example.com/test2");
        indexUpdateCallback.store(paramMap, dataMap2);