diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/analytics/DataImportEndpoints.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/analytics/DataImportEndpoints.java new file mode 100644 index 00000000000..35ae2b24db8 --- /dev/null +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/analytics/DataImportEndpoints.java @@ -0,0 +1,73 @@ +package org.digijava.kernel.ampapi.endpoints.analytics; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.digijava.kernel.ampapi.endpoints.security.AuthRule; +import org.digijava.kernel.ampapi.endpoints.util.ApiMethod; +import org.digijava.kernel.services.analytics.DataImportService; + +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.util.List; +import java.util.Map; + +/** + * Endpoints for fetching data from views for import into another database + */ +@Path("analytics") +@Api("analytics") +public class DataImportEndpoints { + + /** + * Fetches data from the specified views + * + * @param viewNames list of view names to fetch data from + * @return map of view name to list of records + */ + @POST + @Path("/views-data") + @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") + @Consumes(MediaType.APPLICATION_JSON) + @ApiMethod(id = "importViewsData") + @ApiOperation( + value = "Fetch data from views for import", + notes = "Returns data from the specified views in a format suitable for import into another database") + @ApiResponses({ + @ApiResponse(code = HttpServletResponse.SC_OK, message = "Data from views"), + @ApiResponse(code = HttpServletResponse.SC_BAD_REQUEST, message = "Invalid request") + }) + public Map importData( + @ApiParam(value = "List of view names to fetch data from", required = true) + List viewNames) { + + return DataImportService.fetchDataFromViews(viewNames); + } + + /** + * Gets all available views in the database + * + * @return list of view names + */ + @GET + @Path("/views") + @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") + @ApiMethod(id = "getAllViews") + @ApiOperation( + value = "Get all available views", + notes = "Returns a list of all available views in the database") + @ApiResponses({ + @ApiResponse(code = HttpServletResponse.SC_OK, message = "List of views"), + @ApiResponse(code = HttpServletResponse.SC_INTERNAL_SERVER_ERROR, message = "Error fetching views") + }) + public List getAllViews() { + return DataImportService.getAllViews(); + } +} diff --git a/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java b/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java new file mode 100644 index 00000000000..a7ff155ab03 --- /dev/null +++ b/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java @@ -0,0 +1,150 @@ +package org.digijava.kernel.services.analytics; + +import org.dgfoundation.amp.ar.viewfetcher.DatabaseViewFetcher; +import org.dgfoundation.amp.ar.viewfetcher.SQLUtils; +import org.digijava.kernel.persistence.PersistenceManager; +import org.digijava.kernel.request.TLSUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Service for fetching data from database views for import into another database + */ +public class DataImportService { + private static final Logger logger = LoggerFactory.getLogger(DataImportService.class); + + /** + * Fetches data from a list of views + * + * @param viewNames list of view names to fetch data from + * @return map of view name to list of records + */ + public static Map fetchDataFromViews(List viewNames) { + Map result = new LinkedHashMap<>(); + + if (viewNames == null || viewNames.isEmpty()) { + result.put("error", "No views specified"); + return result; + } + + for (String viewName : viewNames) { + try { + List> viewData = fetchViewData(viewName); + result.put(viewName, viewData); + } catch (Exception e) { + logger.error("Error fetching data from view: " + viewName, e); + Map errorInfo = new HashMap<>(); + errorInfo.put("error", e.getMessage()); + result.put(viewName, errorInfo); + } + } + + return result; + } + + /** + * Fetches data from a single view + * + * @param viewName name of the view to fetch data from + * @return list of records from the view + */ + private static List> fetchViewData(String viewName) { + List> records = new ArrayList<>(); + + // Validate that the view exists + if (!viewExists(viewName)) { + throw new IllegalArgumentException("View does not exist: " + viewName); + } + + // Fetch data from the view + PersistenceManager.getSession().doWork(connection -> { + List columns = new ArrayList<>(SQLUtils.getTableColumns(viewName, true)); + + DatabaseViewFetcher.fetchView(connection, TLSUtils.getEffectiveLangCode(), viewName, null, + columns, rs -> { + try { + // No need to call rs.next() here as it's already called in RsInfo.forEach + records.add(resultSetRowToMap(rs)); + } catch (SQLException e) { + throw new RuntimeException("Error processing result set for view: " + viewName, e); + } + }); + }); + + return records; + } + + /** + * Converts a ResultSet row to a Map + * + * @param rs ResultSet to convert + * @return Map representation of the row + * @throws SQLException if there's an error accessing the ResultSet + */ + private static Map resultSetRowToMap(ResultSet rs) throws SQLException { + Map row = new LinkedHashMap<>(); + ResultSetMetaData metaData = rs.getMetaData(); + int columnCount = metaData.getColumnCount(); + + for (int i = 1; i <= columnCount; i++) { + String columnName = metaData.getColumnName(i); + Object value = rs.getObject(i); + row.put(columnName, value); + } + + return row; + } + + /** + * Checks if a view exists in the database + * + * @param viewName name of the view to check + * @return true if the view exists, false otherwise + */ + private static boolean viewExists(String viewName) { + try { + return !SQLUtils.getTableColumns(viewName, true).isEmpty(); + } catch (Exception e) { + return false; + } + } + + /** + * Gets all available views in the database + * + * @return list of view names + */ + public static List getAllViews() { + List views = new ArrayList<>(); + + PersistenceManager.getSession().doWork(connection -> { + try (Statement stmt = connection.createStatement()) { + String query = "SELECT table_name FROM information_schema.tables " + + "WHERE table_schema = 'public' AND table_type = 'VIEW' " + + "ORDER BY table_name"; + + try (ResultSet rs = stmt.executeQuery(query)) { + while (rs.next()) { + views.add(rs.getString("table_name")); + } + } + } catch (SQLException e) { + logger.error("Error fetching views", e); + throw new RuntimeException("Error fetching views", e); + } + }); + + return views; + } +} diff --git a/amp/src/main/resources/xmlpatches/4.0/AMP-31003-Create-Views-For-Superset-import.xml b/amp/src/main/resources/xmlpatches/4.0/AMP-31003-Create-Views-For-Superset-import.xml new file mode 100644 index 00000000000..f5e2a448155 --- /dev/null +++ b/amp/src/main/resources/xmlpatches/4.0/AMP-31003-Create-Views-For-Superset-import.xml @@ -0,0 +1,254 @@ + + + AMP-31003 + bmokandu + Add sample views for superset data import + + + +