/**
     * Sets the delay time in milliseconds for waiting for new URLs.
     *
     * @param delayMillisForWaitingNewUrl the delay time in milliseconds for waiting for new URLs
     */
    public void setDelayMillisForWaitingNewUrl(final long delayMillisForWaitingNewUrl) {
        this.delayMillisForWaitingNewUrl = delayMillisForWaitingNewUrl;
    }

    /**
     * Delays the crawler while waiting for new URLs to be available.