Format     string          `json:"format"`
	URL        xnet.URL        `json:"url"`
	Index      string          `json:"index"`
	QueueDir   string          `json:"queueDir"`
	QueueLimit uint64          `json:"queueLimit"`
	Transport  *http.Transport `json:"-"`
	Username   string          `json:"username"`
	Password   string          `json:"password"`
}

// Validate ElasticsearchArgs fields

ion_string","value":""},{"key":"table","value":""},{"key":"queue_dir","value":""},{"key":"queue_limit","value":"0"},{"key":"max_open_connections","value":"2"}]},"notify_redis":{"_":[{"key":"enable","value":"off"},{"key":"format","value":"namespace"},{"key":"address","value":""},{"key":"key","value":""},{"key":"password","value":""},{"key":"user","value":""},{"key":"queue_dir","value":""},{"key":"queue_limit","value":"0"}]},"notify_webhook":{"_":[{"key":"enable","value":"off"},{"key":"endpoint","...

			Enable:             enabled,
			Brokers:            brokers,
			Topic:              env.Get(topicEnv, kv.Get(target.KafkaTopic)),
			QueueDir:           env.Get(queueDirEnv, kv.Get(target.KafkaQueueDir)),
			QueueLimit:         queueLimit,
			Version:            env.Get(versionEnv, kv.Get(target.KafkaVersion)),
			BatchSize:          uint32(batchSize),
			BatchCommitTimeout: batchCommitTimeout,
		}

		tlsEnableEnv := target.EnvKafkaTLS

			Value: cfg.TLS.ClientTLSCert,
		},
		config.KV{
			Key:   target.KafkaClientTLSKey,
			Value: cfg.TLS.ClientTLSKey,
		},
		config.KV{
			Key:   target.KafkaQueueLimit,
			Value: strconv.Itoa(int(cfg.QueueLimit)),
		},
		config.KV{
			Key:   target.KafkaTLS,
			Value: config.FormatBool(cfg.TLS.Enable),
		},
		config.KV{
			Key:   target.KafkaTLSSkipVerify,
			Value: config.FormatBool(cfg.TLS.SkipVerify),
		},

  private final ConcurrentHashMap<String, AtomicInteger> hostnames = new ConcurrentHashMap<>();
  private final int hostLimit;

  public Crawler(OkHttpClient client, int queueLimit, int hostLimit) {
    this.client = client;
    this.queue = new LinkedBlockingQueue<>(queueLimit);
    this.hostLimit = hostLimit;
  }

  private void parallelDrainQueue(int threadCount) {
    ExecutorService executor = Executors.newFixedThreadPool(threadCount);

type ConsoleLogger struct {
	Enable bool `json:"enable"`
}

// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in all notification targets.
type serverConfigV33 struct {
	quick.Config `json:"-"` // ignore interfaces

	Version string `json:"version"`

	// S3 API configuration.
	Credential auth.Credentials `json:"credential"`