OembedService.java

/*
 * Created by Michael Simons, michael-simons.eu
 * and released under The BSD License
 * http://www.opensource.org/licenses/bsd-license.php
 *
 * Copyright (c) 2010-2026, Michael Simons
 * All rights reserved.
 *
 * Redistribution  and  use  in  source   and  binary  forms,  with  or   without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source   code must retain   the above copyright   notice,
 *   this list of conditions and the following disclaimer.
 *
 * * Redistributions in binary  form must reproduce  the above copyright  notice,
 *   this list of conditions  and the following  disclaimer in the  documentation
 *   and/or other materials provided with the distribution.
 *
 * * Neither the name  of  michael-simons.eu   nor the names  of its contributors
 *   may be used  to endorse   or promote  products derived  from  this  software
 *   without specific prior written permission.
 *
 * THIS SOFTWARE IS  PROVIDED BY THE  COPYRIGHT HOLDERS AND  CONTRIBUTORS "AS IS"
 * AND ANY  EXPRESS OR  IMPLIED WARRANTIES,  INCLUDING, BUT  NOT LIMITED  TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL  THE COPYRIGHT HOLDER OR CONTRIBUTORS  BE LIABLE
 * FOR ANY  DIRECT, INDIRECT,  INCIDENTAL, SPECIAL,  EXEMPLARY, OR  CONSEQUENTIAL
 * DAMAGES (INCLUDING,  BUT NOT  LIMITED TO,  PROCUREMENT OF  SUBSTITUTE GOODS OR
 * SERVICES; LOSS  OF USE,  DATA, OR  PROFITS; OR  BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT  LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE  USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package ac.simons.oembed;

import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.function.Function;
import java.util.stream.Collectors;

import ac.simons.oembed.OembedResponse.Format;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.ehcache.Cache;
import org.ehcache.CacheManager;
import org.ehcache.config.builders.CacheConfigurationBuilder;
import org.ehcache.config.builders.ResourcePoolsBuilder;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.parser.Parser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * This is the main entrpy point for dealing with OEmbed discovery.
 *
 * @author Michael J. Simons
 * @since 2014-12-31
 */
public class OembedService {

	static final Logger LOGGER = LoggerFactory.getLogger(OembedService.class.getPackage().getName());

	/**
	 * This is the http client that will execute all requests.
	 */
	private final HttpClient httpClient;

	/**
	 * An optional cache manager used for caching oembed responses.
	 */
	private final CacheManager cacheManager;

	/**
	 * The user agent to use. We want to be a goot net citizen and provide some info about
	 * us.
	 */
	private final String userAgent;

	/**
	 * An optional application name.
	 */
	private final String applicationName;

	/**
	 * The available parsers. This list isn't changeable.
	 */
	private final Map<Format, OembedParser> parsers;

	/**
	 * All configured endpoints.
	 */
	private final Map<OembedEndpoint, RequestProvider> endpoints;

	/**
	 * All configured renderers. The handlers are grouped by URL schemes.
	 */
	private final Map<List<String>, OembedResponseRenderer> renderers;

	/**
	 * A flag wether autodiscovery of oembed endpoints should be tried. Defaults to false.
	 */
	private boolean autodiscovery = false;

	/**
	 * The name of the cached used by this service. Defaults to the services fully
	 * qualified class name.
	 */
	private String cacheName = OembedService.class.getName();

	/**
	 * Used for auto-discovered endpoints.
	 */
	private final RequestProvider defaultRequestProvider = new DefaultRequestProvider();

	/**
	 * Used for auto-discovered endpoints.
	 */
	private final OembedResponseRenderer defaultRenderer = new DefaultOembedResponseRenderer();

	private final OembedResponseExpiryPolicy expiryPolicy = new OembedResponseExpiryPolicy();

	/**
	 * Creates a new {@code OembedService}. This service depends on a {@link HttpClient}
	 * and can use a {@link CacheManager} for caching requests.
	 * @param httpClient the mandatory http client
	 * @param cacheManager an optional cache manager
	 * @param endpoints the static endpoints
	 * @param applicationName an optional application name
	 */
	public OembedService(final HttpClient httpClient, final CacheManager cacheManager,
			final List<OembedEndpoint> endpoints, final String applicationName) {
		this.httpClient = httpClient;
		this.cacheManager = cacheManager;
		final Properties version = new Properties();
		try {
			version.load(OembedService.class.getResourceAsStream("/oembed.properties"));
		}
		catch (IOException ignored) {
		}
		this.userAgent = String.format("Java/%s java-oembed2/%s", System.getProperty("java.version"),
				version.getProperty("de.dailyfratze.text.oembed.version"));
		this.applicationName = applicationName;

		final Map<Format, OembedParser> hlp = new EnumMap<>(Format.class);
		hlp.put(Format.json, new OembedJsonParser());
		hlp.put(Format.xml, new OembedXmlParser());
		this.parsers = Collections.unmodifiableMap(hlp);

		this.endpoints = endpoints.stream().collect(Collectors.toMap(Function.identity(), endpoint -> {
			LOGGER.debug("Endpoint {} will match the following patterns: {}", endpoint.getName(),
					endpoint.getUrlSchemes());
			LOGGER.debug("Configuring request provider of type {} for endpoint {}...",
					endpoint.getRequestProviderClass(), endpoint.getName());
			LOGGER.debug("Using properties: {}", endpoint.getRequestProviderProperties());

			RequestProvider requestProvider;
			try {
				requestProvider = endpoint.getRequestProviderClass().getDeclaredConstructor().newInstance();
				BeanUtils.populate(requestProvider, endpoint.getRequestProviderProperties());
			}
			catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException
					| InstantiationException ex) {
				// Assuming everything is neatly configured
				throw new OembedException(ex);
			}
			return requestProvider;
		}));

		this.renderers = endpoints.stream().collect(Collectors.toMap(OembedEndpoint::getUrlSchemes, endpoint -> {
			LOGGER.debug("Configuring response renderer of type {} for endpoint {}...",
					endpoint.getResponseRendererClass(), endpoint.getName());
			LOGGER.debug("Using properties: {}", endpoint.getResponseRendererProperties());

			OembedResponseRenderer oembedResponseRenderer = null;
			try {
				oembedResponseRenderer = endpoint.getResponseRendererClass().getDeclaredConstructor().newInstance();
				BeanUtils.populate(oembedResponseRenderer, endpoint.getResponseRendererProperties());
			}
			catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException
					| InstantiationException ex) {
				// Assuming everything is neatly configured
				throw new OembedException(ex);
			}
			return oembedResponseRenderer;
		}));

		LOGGER.debug("Oembed has {} endpoints and autodiscovery {} enabled...", this.endpoints.size(),
				this.autodiscovery ? "is" : "is not");
		LOGGER.info("Oembed ({}) ready...", this.userAgent);
	}

	/**
	 * {@return the current configuration of oembed autodiscovery}
	 */
	public boolean isAutodiscovery() {
		return this.autodiscovery;
	}

	/**
	 * Updates to configuration of oembed autodiscovery.
	 * @param autodiscovery new flag whether oembed endpoints should be auto-discovered
	 */
	public void setAutodiscovery(final boolean autodiscovery) {
		this.autodiscovery = autodiscovery;
	}

	/**
	 * {@return the name of the cached used by this service}
	 */
	public String getCacheName() {
		return this.cacheName;
	}

	/**
	 * Changes the name of the cache used. If a cache manager is present, it clears the
	 * old cache and removes it.
	 * @param cacheName the new cache name
	 */
	public void setCacheName(final String cacheName) {
		if (this.cacheManager != null
				&& this.cacheManager.getCache(this.cacheName, String.class, OembedResponseWrapper.class) != null) {
			this.cacheManager.removeCache(this.cacheName);
		}
		this.cacheName = cacheName;
	}

	/**
	 * {@return the default time in seconds responses are cached}
	 */
	public long getDefaultCacheAge() {
		return this.expiryPolicy.getDefaultCacheAge();
	}

	/**
	 * Changes the default cache age.
	 * @param defaultCacheAge new default cache age in seconds
	 */
	public void setDefaultCacheAge(final long defaultCacheAge) {
		this.expiryPolicy.setDefaultCacheAge(defaultCacheAge);
	}

	/**
	 * Tries to find an endpoint for the given url. It first tries to find an endpoint
	 * within the configured endpoints by a matching url scheme. If that results in an
	 * empty endpoint and auto discovery is enabled, a http GET request is made to the
	 * given url, checking for alternate links with the type
	 * {@code application/(json|xml)+oembed}.
	 * @param url the URL that should be embedded
	 * @return an optional endpoint for this url
	 */
	final Optional<OembedEndpoint> findEndpointFor(final String url) {
		Optional<OembedEndpoint> rv = this.endpoints.keySet()
			.stream()
			.filter(endpoint -> endpoint.getUrlSchemes().stream().map(String::trim).anyMatch(url::matches))
			.findFirst();
		if (rv.isEmpty() && this.autodiscovery) {
			try {
				final HttpResponse httpResponse = this.httpClient.execute(new HttpGet(url));
				if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
					LOGGER.warn("Autodiscovery for {} failed, server returned error {}: {}", url,
							httpResponse.getStatusLine().getStatusCode(),
							EntityUtils.toString(httpResponse.getEntity()));
				}
				else {
					final Document document = Jsoup.parse(EntityUtils.toString(httpResponse.getEntity(), "UTF-8"), url);
					rv = document.getElementsByAttributeValue("rel", "alternate").stream().map(alternate -> {
						OembedEndpoint autodiscoveredEndpoint = null;
						try {
							if (alternate.attr("type").equalsIgnoreCase("application/json+oembed")) {
								autodiscoveredEndpoint = new AutodiscoveredOembedEndpoint(
										new URI(alternate.absUrl("href")), Format.json);
							}
							else if (alternate.attr("type").equalsIgnoreCase("text/xml+oembed")) {
								autodiscoveredEndpoint = new AutodiscoveredOembedEndpoint(
										new URI(alternate.absUrl("href")), Format.xml);
							}
						}
						catch (URISyntaxException ex) {
							// Just ignore them
						}
						return autodiscoveredEndpoint;
					}).filter(Objects::nonNull).findFirst();
				}
			}
			catch (IOException ex) {
				LOGGER.warn("Autodiscovery for {} failed: {}", url, ex.getMessage());
			}
		}
		return rv;
	}

	/**
	 * Executes the given HttpRequest {@code request} and returns an input stream for the
	 * responses content if no error occurred and the server returned a status code OK.
	 * @param request the request to be executed
	 * @return an inputstream to read the response
	 */
	final InputStream executeRequest(final HttpGet request) {
		InputStream rv = null;
		try {
			final HttpResponse httpResponse = this.httpClient.execute(request);
			if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
				LOGGER.warn("Skipping '{}', server returned error {}: {}", request.getURI().toString(),
						httpResponse.getStatusLine().getStatusCode(), EntityUtils.toString(httpResponse.getEntity()));
			}
			else {
				rv = httpResponse.getEntity().getContent();
			}
		}
		catch (IOException ex) {
			LOGGER.warn("Skipping '{}', could not get a response: {}", request.getURI().toString(), ex.getMessage());
		}
		return rv;
	}

	/**
	 * Gets or creates the cache for oembed responses. In Ehcache 3.x, caches need to be
	 * explicitly created with a configuration.
	 * @return the cache instance or null if there is no cache manager
	 */
	private Cache<String, OembedResponseWrapper> getOrCreateCache() {
		if (this.cacheManager == null) {
			return null;
		}
		var cache = this.cacheManager.getCache(this.cacheName, String.class, OembedResponseWrapper.class);
		if (cache != null) {
			return cache;
		}

		return this.cacheManager.createCache(this.cacheName, CacheConfigurationBuilder
			.newCacheConfigurationBuilder(String.class, OembedResponseWrapper.class, ResourcePoolsBuilder.heap(1000))
			.withExpiry(new OembedResponseExpiryPolicy())
			.build());
	}

	/**
	 * Tries to find an {@link OembedResponse} for the URL {@code url}. If a cache manager
	 * is present, it tries that first. If an {@code OembedResponse} can be discovered and
	 * a cache manager is present, that response will be cached.
	 * @param url the URL that might be represented by oembed.
	 * @return an oembed response
	 */
	public Optional<OembedResponse> getOembedResponseFor(final String url) {
		final String trimmedUrl = Optional.ofNullable(url).map(String::trim).orElse("");
		if (trimmedUrl.isEmpty()) {
			LOGGER.debug("Ignoring empty url...");
			return Optional.empty();
		}

		var rv = Optional.ofNullable(this.getOrCreateCache())
			.map(cache -> cache.get(trimmedUrl))
			.map(OembedResponseWrapper::value);
		// If there's already an oembed response cached, use that
		if (rv.isPresent()) {
			LOGGER.debug("Using OembedResponse from cache for '{}'...", trimmedUrl);
			return rv;
		}

		final Optional<OembedEndpoint> endPoint = this.findEndpointFor(trimmedUrl);
		LOGGER.debug("Found endpoint {} for '{}'...", endPoint, trimmedUrl);
		rv = endPoint
			.map(ep -> this.endpoints.getOrDefault(ep, this.defaultRequestProvider)
				.createRequestFor(this.userAgent, this.applicationName, ep.toApiUrl(trimmedUrl)))
			.map(this::executeRequest)
			.map(content -> {
				OembedResponse oembedResponse = null;
				try {
					oembedResponse = this.parsers.get(endPoint.get().getFormat()).unmarshal(content);
				}
				catch (OembedException ex) {
					LOGGER.warn("Server returned an invalid oembed format for url '{}': {}", trimmedUrl,
							ex.getMessage());
				}
				return oembedResponse;
			});

		if (this.cacheManager != null) {
			var cache = getOrCreateCache();
			// We're adding failed urls to the cache as well to prevent them
			// from being tried again over and over (at least for some seconds)
			cache.put(trimmedUrl, new OembedResponseWrapper(rv.orElse(null)));
			LOGGER.debug("Cached response {} from url '{}'...", rv, trimmedUrl);
		}

		return rv;
	}

	/**
	 * Embed all urls found in the given text for which providers are present.
	 * @param textWithEmbeddableUrls text that may contain links
	 * @param baseUrl base url for constructing absolute links
	 * @return a string with urls embedded
	 * @see #embedUrls(java.lang.String, java.lang.String, java.lang.Class)
	 */
	public String embedUrls(final String textWithEmbeddableUrls, final String baseUrl) {
		return embedUrls(textWithEmbeddableUrls, baseUrl, String.class);
	}

	/**
	 * Scans the text {@code textWithEmbeddableUrls} for anchor tags and tries to find
	 * {@link OembedEndpoint} for those urls. If such an endpoint exists, it tries to get
	 * an {@link OembedResponse} of that url from the endpoint. This response will then be
	 * rendered as html and is used to replace the anchor tag.
	 * @param <T> type of the resulting document with embedded links
	 * @param textWithEmbeddableUrls text that contains embeddable urls
	 * @param baseUrl an optional base url for resolving relative urls
	 * @param targetClass the concrete class for the document node
	 * @return the same text with embedded urls if such urls existed
	 */
	@SuppressWarnings("unchecked")
	public <T> T embedUrls(final String textWithEmbeddableUrls, final String baseUrl,
			final Class<? extends T> targetClass) {
		var optionalBaseUrl = Optional.ofNullable(baseUrl);
		T rv;
		if (String.class.isAssignableFrom(targetClass)) {
			rv = (T) textWithEmbeddableUrls;
		}
		else if (Document.class.isAssignableFrom(targetClass)) {
			rv = (T) Document.createShell(optionalBaseUrl.orElse(""));
		}
		else {
			throw new OembedException(String.format("Invalid target class: %s", targetClass.getName()));
		}

		if (!(textWithEmbeddableUrls == null || textWithEmbeddableUrls.trim().isEmpty())) {
			// Create a document
			final Document document = embedUrls(
					Jsoup.parseBodyFragment(textWithEmbeddableUrls, optionalBaseUrl.orElse("")));
			if (Document.class.isAssignableFrom(targetClass)) {
				rv = (T) document;
			}
			else {
				document.outputSettings()
					.prettyPrint(false)
					.escapeMode(EscapeMode.xhtml)
					.charset(StandardCharsets.UTF_8);
				rv = (T) Parser.unescapeEntities(document.body().html().trim(), true);
			}
		}
		return rv;
	}

	/**
	 * A convenience method to embed urls in an existing document.
	 * @param document an existing document, will be modified
	 * @return the modified document with embedded urls
	 * @see #embedUrls(java.lang.String, java.lang.String, java.lang.Class)
	 */
	public Document embedUrls(final Document document) {
		for (Element a : document.getElementsByTag("a")) {
			final String absUrl = a.absUrl("href");
			final Optional<String> html = this.getOembedResponseFor(absUrl).map(response -> {
				final OembedResponseRenderer renderer = this.renderers.entrySet()
					.stream()
					.filter(entry -> entry.getKey().stream().anyMatch(absUrl::matches))
					.findFirst()
					.map(Map.Entry::getValue)
					.orElse(this.defaultRenderer);
				return renderer.render(response, a.clone());
			});
			if (html.isPresent() && !html.get().trim().isEmpty()) {
				a.before(html.get().trim());
				a.remove();
			}
		}
		return document;
	}

	/**
	 * Returns an instance of an {@link OembedParser} for the given
	 * {@link OembedResponse.Format}.
	 * @param format the format for which a parser is needed
	 * @return the parser if a parser for the given format exists
	 */
	public OembedParser getParser(final Format format) {
		return this.parsers.get(format);
	}

}