From a2b91df6bc3fdfebc23e290c455c3d7fc995b049 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Thu, 17 Jul 2025 12:00:59 +0300 Subject: [PATCH 1/6] AMP-31003: Create views for superset data import --- .../analytics/DataImportEndpoints.java | 73 +++++ .../services/analytics/DataImportService.java | 151 +++++++++++ ...31003-Create-Views-For-Superset-import.xml | 254 ++++++++++++++++++ 3 files changed, 478 insertions(+) create mode 100644 amp/src/main/java/org/digijava/kernel/ampapi/endpoints/analytics/DataImportEndpoints.java create mode 100644 amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java create mode 100644 amp/src/main/resources/xmlpatches/4.0/AMP-31003-Create-Views-For-Superset-import.xml diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/analytics/DataImportEndpoints.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/analytics/DataImportEndpoints.java new file mode 100644 index 00000000000..5ba2797c7cd --- /dev/null +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/analytics/DataImportEndpoints.java @@ -0,0 +1,73 @@ +package org.digijava.kernel.ampapi.endpoints.analytics; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.digijava.kernel.ampapi.endpoints.security.AuthRule; +import org.digijava.kernel.ampapi.endpoints.util.ApiMethod; +import org.digijava.kernel.services.analytics.DataImportService; + +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.util.List; +import java.util.Map; + +/** + * Endpoints for fetching data from views for import into another database + */ +@Path("analytics") +@Api("analytics") +public class DataImportEndpoints { + + /** + * Fetches data from the specified views + * + * @param viewNames list of view names to fetch data from + * @return map of view name to list of records + */ + @POST + @Path("/import-data") + @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") + @Consumes(MediaType.APPLICATION_JSON) + @ApiMethod(id = "importViewsData") + @ApiOperation( + value = "Fetch data from views for import", + notes = "Returns data from the specified views in a format suitable for import into another database") + @ApiResponses({ + @ApiResponse(code = HttpServletResponse.SC_OK, message = "Data from views"), + @ApiResponse(code = HttpServletResponse.SC_BAD_REQUEST, message = "Invalid request") + }) + public Map importData( + @ApiParam(value = "List of view names to fetch data from", required = true) + List viewNames) { + + return DataImportService.fetchDataFromViews(viewNames); + } + + /** + * Gets all available views in the database + * + * @return list of view names + */ + @GET + @Path("/views") + @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") + @ApiMethod(id = "getAllViews") + @ApiOperation( + value = "Get all available views", + notes = "Returns a list of all available views in the database") + @ApiResponses({ + @ApiResponse(code = HttpServletResponse.SC_OK, message = "List of views"), + @ApiResponse(code = HttpServletResponse.SC_INTERNAL_SERVER_ERROR, message = "Error fetching views") + }) + public List getAllViews() { + return DataImportService.getAllViews(); + } +} diff --git a/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java b/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java new file mode 100644 index 00000000000..e58185ba15a --- /dev/null +++ b/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java @@ -0,0 +1,151 @@ +package org.digijava.kernel.services.analytics; + +import org.dgfoundation.amp.ar.viewfetcher.DatabaseViewFetcher; +import org.dgfoundation.amp.ar.viewfetcher.SQLUtils; +import org.digijava.kernel.persistence.PersistenceManager; +import org.digijava.kernel.request.TLSUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Service for fetching data from database views for import into another database + */ +public class DataImportService { + private static final Logger logger = LoggerFactory.getLogger(DataImportService.class); + + /** + * Fetches data from a list of views + * + * @param viewNames list of view names to fetch data from + * @return map of view name to list of records + */ + public static Map fetchDataFromViews(List viewNames) { + Map result = new LinkedHashMap<>(); + + if (viewNames == null || viewNames.isEmpty()) { + result.put("error", "No views specified"); + return result; + } + + for (String viewName : viewNames) { + try { + List> viewData = fetchViewData(viewName); + result.put(viewName, viewData); + } catch (Exception e) { + logger.error("Error fetching data from view: " + viewName, e); + Map errorInfo = new HashMap<>(); + errorInfo.put("error", e.getMessage()); + result.put(viewName, errorInfo); + } + } + + return result; + } + + /** + * Fetches data from a single view + * + * @param viewName name of the view to fetch data from + * @return list of records from the view + */ + private static List> fetchViewData(String viewName) { + List> records = new ArrayList<>(); + + // Validate that the view exists + if (!viewExists(viewName)) { + throw new IllegalArgumentException("View does not exist: " + viewName); + } + + // Fetch data from the view + PersistenceManager.getSession().doWork(connection -> { + List columns = new ArrayList<>(SQLUtils.getTableColumns(viewName, true)); + + DatabaseViewFetcher.fetchView(connection, TLSUtils.getEffectiveLangCode(), viewName, null, + columns, rs -> { + try { + while (rs.next()) { + records.add(resultSetRowToMap(rs)); + } + } catch (SQLException e) { + throw new RuntimeException("Error processing result set for view: " + viewName, e); + } + }); + }); + + return records; + } + + /** + * Converts a ResultSet row to a Map + * + * @param rs ResultSet to convert + * @return Map representation of the row + * @throws SQLException if there's an error accessing the ResultSet + */ + private static Map resultSetRowToMap(ResultSet rs) throws SQLException { + Map row = new LinkedHashMap<>(); + ResultSetMetaData metaData = rs.getMetaData(); + int columnCount = metaData.getColumnCount(); + + for (int i = 1; i <= columnCount; i++) { + String columnName = metaData.getColumnName(i); + Object value = rs.getObject(i); + row.put(columnName, value); + } + + return row; + } + + /** + * Checks if a view exists in the database + * + * @param viewName name of the view to check + * @return true if the view exists, false otherwise + */ + private static boolean viewExists(String viewName) { + try { + return !SQLUtils.getTableColumns(viewName, true).isEmpty(); + } catch (Exception e) { + return false; + } + } + + /** + * Gets all available views in the database + * + * @return list of view names + */ + public static List getAllViews() { + List views = new ArrayList<>(); + + PersistenceManager.getSession().doWork(connection -> { + try (Statement stmt = connection.createStatement()) { + String query = "SELECT table_name FROM information_schema.tables " + + "WHERE table_schema = 'public' AND table_type = 'VIEW' " + + "ORDER BY table_name"; + + try (ResultSet rs = stmt.executeQuery(query)) { + while (rs.next()) { + views.add(rs.getString("table_name")); + } + } + } catch (SQLException e) { + logger.error("Error fetching views", e); + throw new RuntimeException("Error fetching views", e); + } + }); + + return views; + } +} diff --git a/amp/src/main/resources/xmlpatches/4.0/AMP-31003-Create-Views-For-Superset-import.xml b/amp/src/main/resources/xmlpatches/4.0/AMP-31003-Create-Views-For-Superset-import.xml new file mode 100644 index 00000000000..f5e2a448155 --- /dev/null +++ b/amp/src/main/resources/xmlpatches/4.0/AMP-31003-Create-Views-For-Superset-import.xml @@ -0,0 +1,254 @@ + + + AMP-31003 + bmokandu + Add sample views for superset data import + + + + From ece9898b3e8d6294f4a8beb5989fac615c676a05 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Thu, 17 Jul 2025 12:23:07 +0300 Subject: [PATCH 2/6] AMP-31003: Create views for superset data import --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index de276d66652..353483163cc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -69,9 +69,9 @@ stage('Build') { println "AMP Version: ${codeVersion}" //Used in the initial generation of keys when working with a new jenkins instance //**************************************************************** -// sh "ssh-keygen -t rsa -b 4096 -C 'jenkins@${environment}' -f ~/.ssh/id_rsa -N ''" + sh "ssh-keygen -t rsa -b 4096 -C 'jenkins@${environment}' -f ~/.ssh/id_rsa -N ''" sh "ssh-keyscan -H ${environment} >> ~/.ssh/known_hosts" -// sh "cat /root/.ssh/id_rsa.pub" + sh "cat /root/.ssh/id_rsa.pub" //****************************************************** countries = sh(returnStdout: true, script: "ssh ${env.jenkinsUser}@${environment} 'cd /opt/amp_dbs && amp-db ls ${codeVersion} | sort'") From e7a3ac0b9c3bfa49e27d45cca517798b9e553240 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Thu, 17 Jul 2025 12:25:32 +0300 Subject: [PATCH 3/6] AMP-31003: Create views for superset data import --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 353483163cc..de276d66652 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -69,9 +69,9 @@ stage('Build') { println "AMP Version: ${codeVersion}" //Used in the initial generation of keys when working with a new jenkins instance //**************************************************************** - sh "ssh-keygen -t rsa -b 4096 -C 'jenkins@${environment}' -f ~/.ssh/id_rsa -N ''" +// sh "ssh-keygen -t rsa -b 4096 -C 'jenkins@${environment}' -f ~/.ssh/id_rsa -N ''" sh "ssh-keyscan -H ${environment} >> ~/.ssh/known_hosts" - sh "cat /root/.ssh/id_rsa.pub" +// sh "cat /root/.ssh/id_rsa.pub" //****************************************************** countries = sh(returnStdout: true, script: "ssh ${env.jenkinsUser}@${environment} 'cd /opt/amp_dbs && amp-db ls ${codeVersion} | sort'") From d6c42323a57943e54b59b2480c150a4788e0b086 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Thu, 17 Jul 2025 13:17:01 +0300 Subject: [PATCH 4/6] AMP-30989: Reopened to correctly change the foreign key --- .../kernel/services/analytics/DataImportService.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java b/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java index e58185ba15a..a7ff155ab03 100644 --- a/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java +++ b/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java @@ -74,9 +74,8 @@ private static List> fetchViewData(String viewName) { DatabaseViewFetcher.fetchView(connection, TLSUtils.getEffectiveLangCode(), viewName, null, columns, rs -> { try { - while (rs.next()) { - records.add(resultSetRowToMap(rs)); - } + // No need to call rs.next() here as it's already called in RsInfo.forEach + records.add(resultSetRowToMap(rs)); } catch (SQLException e) { throw new RuntimeException("Error processing result set for view: " + viewName, e); } From 38e9980cea2c85a4b65f5694ff9396c82441ae50 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Thu, 17 Jul 2025 13:17:01 +0300 Subject: [PATCH 5/6] AMP-31003: Create views for superset data import --- .../kernel/services/analytics/DataImportService.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java b/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java index e58185ba15a..a7ff155ab03 100644 --- a/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java +++ b/amp/src/main/java/org/digijava/kernel/services/analytics/DataImportService.java @@ -74,9 +74,8 @@ private static List> fetchViewData(String viewName) { DatabaseViewFetcher.fetchView(connection, TLSUtils.getEffectiveLangCode(), viewName, null, columns, rs -> { try { - while (rs.next()) { - records.add(resultSetRowToMap(rs)); - } + // No need to call rs.next() here as it's already called in RsInfo.forEach + records.add(resultSetRowToMap(rs)); } catch (SQLException e) { throw new RuntimeException("Error processing result set for view: " + viewName, e); } From c08c37bd3d90bbd722fa7f86c45b8ecac1d5b006 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Fri, 18 Jul 2025 16:18:41 +0300 Subject: [PATCH 6/6] AMP-31003: Create views for superset data import --- .../kernel/ampapi/endpoints/analytics/DataImportEndpoints.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/analytics/DataImportEndpoints.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/analytics/DataImportEndpoints.java index 5ba2797c7cd..35ae2b24db8 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/analytics/DataImportEndpoints.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/analytics/DataImportEndpoints.java @@ -33,7 +33,7 @@ public class DataImportEndpoints { * @return map of view name to list of records */ @POST - @Path("/import-data") + @Path("/views-data") @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") @Consumes(MediaType.APPLICATION_JSON) @ApiMethod(id = "importViewsData")