}
                }
            });
        }

        // Start all threads
        for (Thread thread : threads) {
            thread.start();
        }

        // Wait for all threads to complete
        for (Thread thread : threads) {
            thread.join();
        }

        // Verify no exceptions
        assertTrue(exceptions.isEmpty());
    }

    /**

        }
        return directive.allows(path);
    }

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found

                    }
                }
            });
        }

        // Start all threads
        for (Thread thread : threads) {
            thread.start();
        }

        // Wait for all threads
        for (Thread thread : threads) {
            thread.join();
        }

        // Verify results
        assertTrue(exceptions.isEmpty());

            latch.countDown();
        }).error(t -> {
            t.printStackTrace();
            latch.countDown();
            fail();
        });
        latch.await();
        assertEquals(num, numObInputDoc.get());

        SuggestResponse response = suggester.suggest().setQuery("test").setSuggestDetail(true).execute().getResponse();
        assertEquals(1, response.getNum());
    }