- **Flexible Configuration**: XML-based dependency injection with LastaFlute DI
- **Extensible Architecture**: Plugin system for custom extractors, transformers, and clients
- **Rate Limiting**: Politeness policies and interval controllers
- **URL Filtering**: Regex-based inclusion/exclusion patterns
- **Data Persistence**: Multiple backend options including OpenSearch integration

## Technology Stack

// an enscapsulated writer upto a certain length
// and skip a certain number of bytes.
type LimitWriter struct {
	io.Writer
	skipBytes int64
	wLimit    int64
}

// Write implements the io.Writer interface limiting upto
// configured length, also skips the first N bytes.
func (w *LimitWriter) Write(p []byte) (n int, err error) {
	n = len(p)
	var n1 int
	if w.skipBytes > 0 {
		if w.skipBytes >= int64(len(p)) {

    /**
     * {@return all source root directories}, including the disabled ones, for all languages and scopes.
     * For listing only the {@linkplain SourceRoot#enabled() enabled} source roots,
     * the following code can be used:
     *
     * <pre>{@literal
     * List<SourceRoot> enabledRoots = project.getSourceRoots()

				},
			)
			if err == nil || errors.Is(err, context.Canceled) {
				return
			}
			setN := humanize.Ordinal(setIdx + 1)
			rebalanceLogIf(ctx, fmt.Errorf("listing objects from %s set failed with %v", setN, err), "rebalance-listing-failed"+setN)
		}(setIdx)
	}

	wk.Wait()
	return nil
}

type rebalSaveOpts uint8

const (
	rebalSaveStats rebalSaveOpts = iota
	rebalSaveStoppedAt

			}
		}
		// loop through buckets and see if some with lost quorum
		// these could be stale buckets lying around, queue a heal
		// of such a bucket. This is needed here as we identify such
		// buckets here while listing buckets. As part of regular
		// globalBucketMetadataSys.Init() call would get a valid
		// buckets only and not the quourum lost ones like this, so
		// explicit call
		for bktName, count := range bucketsMap {

    if (strictParsing) {
      checkState(
          this.maximumSize == UNSET_INT,
          "weigher can not be combined with maximum size (%s provided)",
          this.maximumSize);
    }

    // safely limiting the kinds of caches this can produce
    @SuppressWarnings("unchecked")
    CacheBuilder<K1, V1> me = (CacheBuilder<K1, V1>) this;
    me.weigher = checkNotNull(weigher);
    return me;
  }

     *         workgroups, servers, or shares depending on the context of the
     *         resource URL
     * @throws SmbException if an error occurs while listing the contents
     */
    public String[] list() throws SmbException {
        return SmbEnumerationUtil.list(this, "*", ATTR_DIRECTORY | ATTR_HIDDEN | ATTR_SYSTEM, null, null);
    }

    /**

 *   <li>Content length validation</li>
 *   <li>MIME type detection</li>
 *   <li>Handling of large files through temporary file storage</li>
 *   <li>Object metadata and tags retrieval</li>
 *   <li>Directory listing capabilities</li>
 * </ul>
 *
 * <p>The client handles timeout management for access operations and includes proper
 * resource cleanup mechanisms.
 */
public class S3Client extends AbstractCrawlerClient {

        if (f1.isDirectory()) {
            return f1name.compareToIgnoreCase(f2.getName());
        }
        return f1.lastModified() > f2.lastModified() ? -1 : 1;
    }

    /**
     * Handles directory listing requests for SMB directories.
     * @param req the HTTP servlet request
     * @param resp the HTTP servlet response
     * @param dir the SMB directory to list
     * @throws IOException if an I/O error occurs
     */

 * to interact with SMB resources.
 *
 * <p>
 * This client supports authentication, content retrieval, and metadata extraction from SMB files.
 * It handles file access, directory listing, and access control entries (ACEs) processing.
 * </p>
 *
 * <p>
 * The class provides methods to:
 * </p>
 * <ul>
 *   <li>Initialize the client with SMB authentication details.</li>