---

## Key Components

### Crawler (`Crawler.java`)

Main orchestrator for crawling operations.

**Key Methods**:
```java
String execute()                // Start crawling, return session ID
void addUrl(String url)         // Add URL to queue
void cleanup(String sessionId)  // Clean up session
void stop()                     // Stop gracefully
```

```java
// Create multiple crawler instances
Crawler crawler1 = container.getComponent("crawler");
crawler1.setSessionId("session1");
crawler1.addUrl("https://site1.com");

Crawler crawler2 = container.getComponent("crawler");  
crawler2.setSessionId("session2");
crawler2.addUrl("https://site2.com");

// Execute concurrently
crawler1.setBackground(true);
crawler2.setBackground(true);

			}
		}
	}
}

func TestSameLocalAddrs(t *testing.T) {
	testCases := []struct {
		addr1       string
		addr2       string
		sameAddr    bool
		expectedErr error
	}{
		{"", "", false, errors.New("unable to process empty address")},
		{":9000", ":9000", true, nil},
		{"localhost:9000", ":9000", true, nil},

                addr = new NbtAddress(name, ip, false, NbtAddress.B_NODE, false, false, true, true, NbtAddress.UNKNOWN_MAC_ADDRESS);
                if (log.isDebugEnabled()) {
                    log.debug("Adding " + name + " with addr " + addr);
                }
                this.table.put(name, addr);
            }
        }
    }

		}).DialContext
	}

	return func(ctx context.Context, network, addr string) (conn net.Conn, err error) {
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			return nil, err
		}

		if net.ParseIP(host) != nil {
			// For IP only setups there is no need for DNS lookups.
			return baseDialCtx(ctx, "tcp", addr)
		}

		ips, err := lookupHost(ctx, host)
		if err != nil {
			return nil, err

		haddrs, err := globalDNSCache.LookupHost(ctx, host)
		if err == nil {
			for _, addr := range haddrs {
				addrs.Add(net.JoinHostPort(addr, globalMinioPort))
			}
		} else {
			// Unable to lookup host in 2-secs, let it fail later anyways.
			addrs.Add(globalMinioAddr)
		}
	} else {
		addrs.Add(globalMinioAddr)
	}
	return addrs.ToSlice()
}

var globalLoggerOutput io.WriteCloser

import org.codelibs.fess.crawler.service.UrlQueueService;

import jakarta.annotation.Resource;

/**
 * The Crawler class is the main class for web crawling. It manages the crawling process,
 * including adding URLs to the queue, filtering URLs, managing crawler threads,
 * and handling the overall crawling lifecycle.
 *
 * <p>It implements the Runnable interface to be executed in a separate thread,

 * It also includes methods to build and process the request using OpenSearch's client.</p>
 *
 * <p>Key functionalities include:</p>
 * <ul>
 *   <li>Setting the index, query, and size of the request.</li>
 *   <li>Adding tags, roles, fields, kinds, and languages to the request.</li>
 *   <li>Configuring suggestion details, reading converter, normalizer, prefix match weight, match word first, and skip duplicate words options.</li>

            file.delete();
            file.mkdirs();
            file.deleteOnExit();
            fileTransformer.setPath(file.getAbsolutePath());
            crawler.addUrl(url);
            crawler.crawlerContext.setMaxAccessCount(maxCount);
            crawler.crawlerContext.setNumOfThread(numOfThread);
            crawler.urlFilter.addInclude(url + ".*");

	for _, test := range tests {
		parser.start(lex.Tokenize(test.input))
		addr := obj.Addr{}
		parser.operand(&addr)
		var result string
		if parser.allowABI {
			result = obj.DconvWithABIDetail(&emptyProg, &addr)
		} else {
			result = obj.Dconv(&emptyProg, &addr)
		}
		if result != test.output {
			t.Errorf("fail at %s: got %s; expected %s\n", test.input, result, test.output)
		}