file.delete();
            file.mkdirs();
            file.deleteOnExit();
            fileTransformer.setPath(file.getAbsolutePath());
            crawler.addUrl(url);
            crawler.crawlerContext.setMaxAccessCount(maxCount);
            crawler.crawlerContext.setNumOfThread(numOfThread);
            crawler.urlFilter.addInclude(url + ".*");

            file.delete();
            file.mkdirs();
            file.deleteOnExit();
            fileTransformer.setPath(file.getAbsolutePath());
            crawler.addUrl(url);
            crawler.crawlerContext.setMaxAccessCount(maxCount);
            crawler.crawlerContext.setNumOfThread(numOfThread);
            crawler.urlFilter.addInclude(url + ".*");

```java
// Create multiple crawler instances
Crawler crawler1 = container.getComponent("crawler");
crawler1.setSessionId("session1");
crawler1.addUrl("https://site1.com");

Crawler crawler2 = container.getComponent("crawler");  
crawler2.setSessionId("session2");
crawler2.addUrl("https://site2.com");

// Execute concurrently
crawler1.setBackground(true);
crawler2.setBackground(true);

 * the maximum depth of crawling, and URL filters.
 *
 * <p>Example usage:
 * <pre>
 *   Crawler crawler = new Crawler();
 *   crawler.addUrl("http://example.com/");
 *   crawler.execute();
 *   crawler.close();
 * </pre>
 */
public class Crawler implements Runnable, AutoCloseable {

                if (!urlValue.startsWith("#") && protocolHelper.isValidWebProtocol(urlValue)) {
                    final String u = duplicateHostHelper.convert(urlValue);
                    crawler.addUrl(u);
                    if (logger.isInfoEnabled()) {
                        logger.info("Target URL: {}", u);
                    }
                }
            }));

            // set included urls

            } else {
                logger.warn("urlConvertHelper is null.");
                redirectUrl = redirectUrlObj.toString();
            }
            resultData.addUrl(RequestDataBuilder.newRequestData().get().url(redirectUrl).build());
        }

        return resultData;
    }

    /**
     * Checks if the response data represents HTML content.
     *

            final boolean forceSigning) throws IOException {

        final Address[] addrs = tf.getNameServiceClient().getAllByName(name, true);

        if (addrs == null || addrs.length == 0) {
            throw new UnknownHostException(name);
        }

        Arrays.sort(addrs, (o1, o2) -> {
            Integer fail1 = SmbTransportPoolImpl.this.failCounts.get(o1.getHostAddress());

        } else if (calledName == SMBSERVER_NAME) {
            NbtAddress[] addrs;

            try {
                addrs = CLIENT.getNodeStatus(this);
                if (hostName.hexCode == 0x1D) {
                    for (final NbtAddress addr : addrs) {
                        if (addr.hostName.hexCode == 0x20) {
                            return addr.hostName.name;
                        }
                    }

        Object addr;
        int i;

        if (hostname == null || hostname.length() == 0) {
            throw new UnknownHostException();
        }

        if (isDotQuadIP(hostname)) {
            final UniAddress[] addrs = new UniAddress[1];
            addrs[0] = new UniAddress(NbtAddress.getByName(hostname));
            return addrs;
        }

        Address addr1 = mock(Address.class);
        when(addr1.getHostAddress()).thenReturn("10.0.0.1");

        Address addr2 = mock(Address.class);
        when(addr2.getHostAddress()).thenReturn("10.0.0.2");

        when(nameSvc.getAllByName(eq("test.server"), eq(true))).thenReturn(new Address[] { addr1, addr2 });

        // Set fail counts (addr1 has more failures, addr2 has fewer)