From 88ec7c26f79deb30126967cfbeb9eaeafc8464ff Mon Sep 17 00:00:00 2001 From: Gary O'Neall Date: Sat, 25 Oct 2025 20:42:50 -0700 Subject: [PATCH] Update OSI API Fixes #224 --- Test/org/spdx/crossref/OsiApiTest.java | 4 +- src/org/spdx/crossref/OsiApi.java | 60 ++++++++++---------- src/org/spdx/crossref/OsiLicense.java | 78 ++++++++++++-------------- 3 files changed, 69 insertions(+), 73 deletions(-) diff --git a/Test/org/spdx/crossref/OsiApiTest.java b/Test/org/spdx/crossref/OsiApiTest.java index f3eeef5..664d0ce 100644 --- a/Test/org/spdx/crossref/OsiApiTest.java +++ b/Test/org/spdx/crossref/OsiApiTest.java @@ -61,7 +61,7 @@ public void test() throws InvalidSPDXAnalysisException { SpdxListedLicense apache20 = new SpdxListedLicense("Apache-2.0"); apache20.setLicenseText("Apache text"); CrossRef osiApacheCrossRef = new CrossRef(); - String apache20OsiUrl = "https://opensource.org/licenses/Apache-2.0"; + String apache20OsiUrl = "https://opensource.org/licenses/apache-2.0"; osiApacheCrossRef.setUrl(apache20OsiUrl); OsiApi instance = OsiApi.getInstance(); assertTrue(instance.isApiAvailable()); @@ -75,7 +75,7 @@ public void test() throws InvalidSPDXAnalysisException { assertTrue(osiApacheCrossRef.getValid().get()); // not OSI URL - String nonOsiUrl = "https://notopensource.org/licenses/Apache-2.0"; + String nonOsiUrl = "https://notopensource.org/licenses/apache-2.0"; assertFalse(OsiApi.isOsiUrl(nonOsiUrl)); // not matching URL diff --git a/src/org/spdx/crossref/OsiApi.java b/src/org/spdx/crossref/OsiApi.java index 237cb1e..fdccc36 100644 --- a/src/org/spdx/crossref/OsiApi.java +++ b/src/org/spdx/crossref/OsiApi.java @@ -51,13 +51,12 @@ */ public class OsiApi { static final Logger logger = LoggerFactory.getLogger(OsiApi.class.getName()); - public static final String OSI_PREFIX = "https://opensource.org/licenses"; - private static final String API_BASE_URL = "https://api.opensource.org"; - private static final String ALL_LICENSES_URL = API_BASE_URL + "/licenses/"; + public static final String OSI_PREFIX = "https://opensource.org/license"; + private static final String API_BASE_URL = "https://opensource.org/api"; + private static final String ALL_LICENSES_URL = API_BASE_URL + "/license"; private static final int READ_TIMEOUT = 5000; - static final List WHITE_LIST = Collections.unmodifiableList(Arrays.asList( - "osi.org")); // currently, we're not allowing any redirects to sites other than OSI - + static final List WHITE_LIST = Collections.unmodifiableList(List.of( + "osi.org")); // currently, we're not allowing any redirects to sites other than OSI private static class InstanceHolder { private static final OsiApi INSTANCE = new OsiApi(); } @@ -71,7 +70,19 @@ public static boolean isOsiUrl(String url) { } private boolean apiAvailable = false; - private Map> urlToSpdxIds = new HashMap<>(); + private final Map urlToSpdxId = new HashMap<>(); + + /** + * Normalized an OSI URL taking into account changes and redirects introduced by OSI which breaks + * compatibility with non-normalized URLs + * @param url URL pointing to an OSI license + * @return normalized URL + */ + public static String normalizeOsiUrl(String url) { + return url.toLowerCase() // OSI changed the case on URLs like Apache-2.0 (now apache-2.0) + .replace("/licenses/", "/license/") // OSI changed the URLs for licenses removing the 's' + .replaceAll("(\\d)\\.(\\d)$", "$1-$2"); // OSI changed the URL ending in apache-2.0 to apache-2-0 + } private OsiApi() { try (BufferedReader reader = new BufferedReader( @@ -87,22 +98,13 @@ private OsiApi() { for (OsiLicense osiLicense:osiLicenses) { if (Objects.nonNull(osiLicense.getLinks()) && Objects.nonNull(osiLicense.getId()) && - Objects.nonNull(osiLicense.getIdentifiers())) { - List spdxIds = new ArrayList<>(); - for (OsiLicense.IdentifierType identifier:osiLicense.getIdentifiers()) { - if ("SPDX".equals(identifier.getScheme()) && - Objects.nonNull(identifier.getIdentifier())) { - spdxIds.add(identifier.getIdentifier()); - } - } - if (!spdxIds.isEmpty()) { - for (OsiLicense.Link link:osiLicense.getLinks()) { - if (Objects.nonNull(link.getUrl())) { - urlToSpdxIds.put(link.getUrl(), spdxIds); - } - } - } - + Objects.nonNull(osiLicense.getSpdx_id())) { + if (Objects.nonNull(osiLicense.getLinks().getSelf()) && Objects.nonNull(osiLicense.getLinks().getSelf().getHref())) { + urlToSpdxId.put(normalizeOsiUrl(osiLicense.getLinks().getSelf().getHref()), osiLicense.getSpdx_id()); + } + if (Objects.nonNull(osiLicense.getLinks().getHtml()) && Objects.nonNull(osiLicense.getLinks().getHtml().getHref())) { + urlToSpdxId.put(normalizeOsiUrl(osiLicense.getLinks().getHtml().getHref()), osiLicense.getSpdx_id()); + } } } apiAvailable = true; @@ -123,9 +125,8 @@ private InputStream getUrlInputStream(URL url) throws IOException { HttpURLConnection connection = (HttpURLConnection)url.openConnection(); connection.setReadTimeout(READ_TIMEOUT); int status = connection.getResponseCode(); - if (status != HttpURLConnection.HTTP_OK && - (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM - || status == HttpURLConnection.HTTP_SEE_OTHER)) { + if ((status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM + || status == HttpURLConnection.HTTP_SEE_OTHER)) { // redirect String redirectUrlStr = connection.getHeaderField("Location"); if (Objects.isNull(redirectUrlStr) || redirectUrlStr.isEmpty()) { @@ -163,9 +164,10 @@ public boolean isApiAvailable() { */ public void setCrossRefDetails(String url, SpdxListedLicense license, CrossRef crossRef) throws InvalidSPDXAnalysisException { - Boolean isValidUrl = Valid.urlValidator(url); - List spdxIds = urlToSpdxIds.get(url); - Boolean isLiveUrl = Objects.nonNull(spdxIds) && spdxIds.contains(license.getId()); + String normalizedUrl = normalizeOsiUrl(url); + Boolean isValidUrl = Valid.urlValidator(normalizedUrl); + String spdxId = urlToSpdxId.get(normalizedUrl); + Boolean isLiveUrl = Objects.equals(spdxId, license.getId()); Boolean isWaybackUrl = false; String currentDate = Timestamp.getTimestamp(); String matchStatus = "N/A"; diff --git a/src/org/spdx/crossref/OsiLicense.java b/src/org/spdx/crossref/OsiLicense.java index a41c1b0..c98f665 100644 --- a/src/org/spdx/crossref/OsiLicense.java +++ b/src/org/spdx/crossref/OsiLicense.java @@ -27,49 +27,40 @@ * */ public class OsiLicense { - - public class IdentifierType { - String identifier; - String scheme; - /** - * @return the identifier - */ - public String getIdentifier() { - return identifier; - } - /** - * @return the scheme - */ - public String getScheme() { - return scheme; + + public static class LinkType { + String href; + + public String getHref() { + return href; } - } - - public class Link { - String note; - String url; - /** - * @return the note - */ - public String getNote() { - return note; + + public static class Links { + LinkType self; + LinkType html; + LinkType collection; + + public LinkType getSelf() { + return self; + } + + public LinkType getHtml() { + return html; } - /** - * @return the url - */ - public String getUrl() { - return url; + + public LinkType getCollection() { + return collection; } - - } String id; - List identifiers; - List links; + Links _links; String name; + String spdx_id; List keywords; + String version; + String submission_date; /** * @return the id */ @@ -79,14 +70,8 @@ public String getId() { /** * @return the identifiers */ - public List getIdentifiers() { - return identifiers; - } - /** - * @return the links - */ - public List getLinks() { - return links; + public Links getLinks() { + return _links; } /** * @return the name @@ -100,4 +85,13 @@ public String getName() { public List getKeywords() { return keywords; } + public String getSpdx_id() { + return spdx_id; + } + public String getVersion() { + return version; + } + public String getSubmission_date() { + return submission_date; + } }