* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.transformer;

import static org.codelibs.core.stream.StreamUtil.stream;

import java.io.InputStream;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;

                accessResult.setAccessResultData(null);
                try {
                    final Transformer transformer = ComponentUtil.getComponent(accessResultData.getTransformerName());
                    if (transformer == null) {
                        // no transformer
                        logger.warn("No transformer: {}", accessResultData.getTransformerName());
                        continue;
                    }

                if (responseProcessor instanceof DefaultResponseProcessor) {
                    final Transformer transformer = ((DefaultResponseProcessor) responseProcessor).getTransformer();
                    final ResultData resultData = transformer.transform(responseData);
                    final byte[] data = resultData.getData();
                    if (data != null) {

        assertTrue(transformer.isValidUrl("http://www.example.com"));
        assertTrue(transformer.isValidUrl("http://www.example.com/aaa"));
        assertTrue(transformer.isValidUrl("https://www.example.com"));
        assertTrue(transformer.isValidUrl("://www.example.com"));
        assertTrue(transformer.isValidUrl("//www.example.com"));

        assertFalse(transformer.isValidUrl(null));

                        + responseProcessor + ", url: " + url);
            }
            final Transformer transformer = ((DefaultResponseProcessor) responseProcessor).getTransformer();
            final ResultData resultData = transformer.transform(responseData);
            final byte[] data = resultData.getData();
            if (data != null) {
                try {

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.transformer;

import static org.codelibs.core.stream.StreamUtil.stream;

import java.io.BufferedInputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;