From e3b5e7d7cf1692376af61df2001d9d4655544d55 Mon Sep 17 00:00:00 2001 From: "Hosea, Elizabeth K. [AU-AU]" Date: Thu, 14 Aug 2025 12:35:21 +1000 Subject: [PATCH 1/9] Fix Large file uploads overflowing heap space --- ui-backend/catalog-ui-search/pom.xml | 5 + .../ui/catalog/CatalogApplication.java | 133 +++++++++--------- .../ui/catalog/CatalogCxfApplication.java | 45 ++++++ .../OSGI-INF/blueprint/blueprint.xml | 8 ++ .../OSGI-INF/blueprint/endpoints.xml | 12 ++ 5 files changed, 137 insertions(+), 66 deletions(-) create mode 100644 ui-backend/catalog-ui-search/src/main/java/org/codice/ddf/catalog/ui/catalog/CatalogCxfApplication.java diff --git a/ui-backend/catalog-ui-search/pom.xml b/ui-backend/catalog-ui-search/pom.xml index c4c85cf8d65..b812356cf71 100644 --- a/ui-backend/catalog-ui-search/pom.xml +++ b/ui-backend/catalog-ui-search/pom.xml @@ -390,6 +390,11 @@ catalog-rest-api ${ddf.version} + + ddf.catalog.rest + catalog-rest-endpoint + ${ddf.version} + com.google.guava guava diff --git a/ui-backend/catalog-ui-search/src/main/java/org/codice/ddf/catalog/ui/catalog/CatalogApplication.java b/ui-backend/catalog-ui-search/src/main/java/org/codice/ddf/catalog/ui/catalog/CatalogApplication.java index a70bd38719e..d82d29d1e6e 100644 --- a/ui-backend/catalog-ui-search/src/main/java/org/codice/ddf/catalog/ui/catalog/CatalogApplication.java +++ b/ui-backend/catalog-ui-search/src/main/java/org/codice/ddf/catalog/ui/catalog/CatalogApplication.java @@ -17,14 +17,12 @@ import static spark.Spark.get; import static spark.Spark.head; import static spark.Spark.post; -import static spark.Spark.put; import com.google.common.collect.ImmutableList; import ddf.catalog.data.BinaryContent; import ddf.catalog.data.Metacard; import ddf.catalog.resource.DataUsageLimitExceededException; import ddf.catalog.resource.Resource; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -32,7 +30,6 @@ import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.util.List; -import javax.servlet.MultipartConfigElement; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.InternalServerErrorException; import javax.ws.rs.core.MediaType; @@ -50,6 +47,7 @@ import org.slf4j.LoggerFactory; import spark.Request; import spark.Response; +import spark.Spark; import spark.servlet.SparkApplication; public class CatalogApplication implements SparkApplication { @@ -80,6 +78,8 @@ public CatalogApplication(CatalogService catalogService) { @Override public void init() { + + LOGGER.info("PATPAT: port", Spark.port()); head( "/catalog/", (req, res) -> { @@ -140,69 +140,70 @@ public void init() { } }); - post( - "/catalog/", - (req, res) -> { - if (req.contentType().startsWith("multipart/")) { - req.attribute( - ECLIPSE_MULTIPART_CONFIG, - new MultipartConfigElement(System.getProperty(JAVA_IO_TMPDIR))); - - return addDocument( - res, - req.raw().getRequestURL(), - req.contentType(), - req.queryParams(TRANSFORM), - req.raw(), - new ByteArrayInputStream(req.bodyAsBytes())); - } - - if (req.contentType().startsWith("text/") - || req.contentType().startsWith("application/")) { - return addDocument( - res, - req.raw().getRequestURL(), - req.contentType(), - req.queryParams(TRANSFORM), - null, - new ByteArrayInputStream(req.bodyAsBytes())); - } - - res.status(HttpStatus.SC_NOT_FOUND); - return res; - }); - - put( - CATALOG_ID_PATH, - (req, res) -> { - if (req.contentType().startsWith("multipart/")) { - req.attribute( - ECLIPSE_MULTIPART_CONFIG, - new MultipartConfigElement(System.getProperty(JAVA_IO_TMPDIR))); - - return updateDocument( - res, - req.params(":id"), - req.contentType(), - req.queryParams(TRANSFORM), - req.raw(), - new ByteArrayInputStream(req.bodyAsBytes())); - } - - if (req.contentType().startsWith("text/") - || req.contentType().startsWith("application/")) { - return updateDocument( - res, - req.params(":id"), - req.contentType(), - req.queryParams(TRANSFORM), - null, - new ByteArrayInputStream(req.bodyAsBytes())); - } - - res.status(HttpStatus.SC_NOT_FOUND); - return res; - }); + // post( + // "/catalog/", + // (req, res) -> { + // LOGGER.info("PATPAT: port", Spark.port()); + // if (req.contentType().startsWith("multipart/")) { + // req.attribute( + // ECLIPSE_MULTIPART_CONFIG, + // new MultipartConfigElement(System.getProperty(JAVA_IO_TMPDIR))); + // + // return addDocument( + // res, + // req.raw().getRequestURL(), + // req.contentType(), + // req.queryParams(TRANSFORM), + // req.raw(), + // new ByteArrayInputStream(req.bodyAsBytes())); + // } + // + // if (req.contentType().startsWith("text/") + // || req.contentType().startsWith("application/")) { + // return addDocument( + // res, + // req.raw().getRequestURL(), + // req.contentType(), + // req.queryParams(TRANSFORM), + // null, + // new ByteArrayInputStream(req.bodyAsBytes())); + // } + // + // res.status(HttpStatus.SC_NOT_FOUND); + // return res; + // }); + // + // put( + // CATALOG_ID_PATH, + // (req, res) -> { + // if (req.contentType().startsWith("multipart/")) { + // req.attribute( + // ECLIPSE_MULTIPART_CONFIG, + // new MultipartConfigElement(System.getProperty(JAVA_IO_TMPDIR))); + // + // return updateDocument( + // res, + // req.params(":id"), + // req.contentType(), + // req.queryParams(TRANSFORM), + // req.raw(), + // new ByteArrayInputStream(req.bodyAsBytes())); + // } + // + // if (req.contentType().startsWith("text/") + // || req.contentType().startsWith("application/")) { + // return updateDocument( + // res, + // req.params(":id"), + // req.contentType(), + // req.queryParams(TRANSFORM), + // null, + // new ByteArrayInputStream(req.bodyAsBytes())); + // } + // + // res.status(HttpStatus.SC_NOT_FOUND); + // return res; + // }); delete( CATALOG_ID_PATH, diff --git a/ui-backend/catalog-ui-search/src/main/java/org/codice/ddf/catalog/ui/catalog/CatalogCxfApplication.java b/ui-backend/catalog-ui-search/src/main/java/org/codice/ddf/catalog/ui/catalog/CatalogCxfApplication.java new file mode 100644 index 00000000000..20a1e62ed37 --- /dev/null +++ b/ui-backend/catalog-ui-search/src/main/java/org/codice/ddf/catalog/ui/catalog/CatalogCxfApplication.java @@ -0,0 +1,45 @@ +package org.codice.ddf.catalog.ui.catalog; + +import java.io.InputStream; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.*; +import org.apache.cxf.jaxrs.ext.multipart.MultipartBody; +import org.codice.ddf.endpoints.rest.RESTEndpoint; + +@Path("/catalog") +public class CatalogCxfApplication { + + private final RESTEndpoint restEndpoint; + + public CatalogCxfApplication(RESTEndpoint restEndpoint) { + this.restEndpoint = restEndpoint; + } + + @POST + @Consumes({"text/*", "application/*"}) + @Produces(MediaType.APPLICATION_JSON) + public Response addDocument( + @Context HttpHeaders headers, + @Context UriInfo requestUriInfo, + @Context HttpServletRequest httpRequest, + @QueryParam("transform") String transformerParam, + InputStream message) { + return restEndpoint.addDocument( + headers, requestUriInfo, httpRequest, transformerParam, message); + } + + @POST + @Consumes("multipart/*") + @Produces(MediaType.APPLICATION_JSON) + public Response addDocument( + @Context HttpHeaders headers, + @Context UriInfo requestUriInfo, + @Context HttpServletRequest httpRequest, + MultipartBody multipartBody, + @QueryParam("transform") String transformerParam, + InputStream message) { + return restEndpoint.addDocument( + headers, requestUriInfo, httpRequest, multipartBody, transformerParam, message); + } +} diff --git a/ui-backend/catalog-ui-search/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/ui-backend/catalog-ui-search/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 390132c9228..35f5332e33d 100644 --- a/ui-backend/catalog-ui-search/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/ui-backend/catalog-ui-search/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -149,6 +149,14 @@ Implementation details + + + + + + + + diff --git a/ui-backend/catalog-ui-search/src/main/resources/OSGI-INF/blueprint/endpoints.xml b/ui-backend/catalog-ui-search/src/main/resources/OSGI-INF/blueprint/endpoints.xml index 9bab1d2bbb6..01116db9fb4 100644 --- a/ui-backend/catalog-ui-search/src/main/resources/OSGI-INF/blueprint/endpoints.xml +++ b/ui-backend/catalog-ui-search/src/main/resources/OSGI-INF/blueprint/endpoints.xml @@ -45,6 +45,18 @@ Services (HTTP/WS) that catalog-ui-search provides for the UI + + + + + + + + - - - - - - -