From 567e60c9514d21a1def5c3d0bfeed1ad341757dc Mon Sep 17 00:00:00 2001 From: Antonio Contreras Date: Fri, 6 Feb 2026 18:22:43 +0100 Subject: [PATCH 1/4] work in progress with the integration --- functions/src/functions/scraping-functions.ts | 11 ++++ .../scraping-cuenca-cantabrico/package.json | 12 +++- .../scraping-cuenca-cantabrico/src/index.ts | 1 + .../src/integration.ts | 6 +- .../src/scraper/business.ts | 2 +- .../src/scraper/mapper.ts | 2 +- .../scraping-cuenca-cantabrico/tsconfig.json | 39 +++++-------- .../scraping-cuenca-mediterranea/package.json | 2 +- package-lock.json | 9 +++ packages/db/package.json | 3 +- .../db/src/dals/embalses/embalses.mappers.ts | 20 +++++++ .../src/dals/embalses/embalses.repository.ts | 56 ++++++++++++++++++- 12 files changed, 128 insertions(+), 35 deletions(-) create mode 100644 integrations/scraping-cuenca-cantabrico/src/index.ts diff --git a/functions/src/functions/scraping-functions.ts b/functions/src/functions/scraping-functions.ts index 7ab68f4..98ca007 100644 --- a/functions/src/functions/scraping-functions.ts +++ b/functions/src/functions/scraping-functions.ts @@ -11,6 +11,8 @@ export async function scrapingsFunction( const responseCuencaMediterranea = await embalsesRepository.actualizarCuencaMediterranea(); + const responseCuencaCantabrico = await embalsesRepository.actualizarCuencaCantabrico(); + if (responseCuencaMediterranea) { context.log(`Se han actualizado los embalses de la cuenca Mediterránea`); } else { @@ -18,6 +20,15 @@ export async function scrapingsFunction( "No se han podido actualizar los embalses de la cuenca Mediterránea" ); } + + if (responseCuencaCantabrico) { + context.log(`Se han actualizado los embalses de la cuenca Cantábrico`); + } else { + context.log( + "No se han podido actualizar los embalses de la cuenca Cantábrico" + ); + } + await dbServer.disconnect(); } diff --git a/integrations/scraping-cuenca-cantabrico/package.json b/integrations/scraping-cuenca-cantabrico/package.json index fc4ad41..12467cf 100644 --- a/integrations/scraping-cuenca-cantabrico/package.json +++ b/integrations/scraping-cuenca-cantabrico/package.json @@ -4,12 +4,18 @@ "private": true, "type": "module", "exports": { - ".": "./src/index.ts" + ".": "./dist/index.js" }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", "scripts": { - "start": "tsx --watch src/console-runner.ts" + "start": "tsx --watch src/console-runner.ts", + "build": "run-p clean type-check build:scraping-cuenca-cantabrico", + "build:scraping-cuenca-cantabrico": "tsc", + "clean": "rimraf dist", + "type-check": "tsc --noEmit --preserveWatchOutput" }, "dependencies": { "db-model": "^1.0.0" } -} \ No newline at end of file +} diff --git a/integrations/scraping-cuenca-cantabrico/src/index.ts b/integrations/scraping-cuenca-cantabrico/src/index.ts new file mode 100644 index 0000000..ba7a4ee --- /dev/null +++ b/integrations/scraping-cuenca-cantabrico/src/index.ts @@ -0,0 +1 @@ +export * from "./integration.js"; diff --git a/integrations/scraping-cuenca-cantabrico/src/integration.ts b/integrations/scraping-cuenca-cantabrico/src/integration.ts index 2463fdb..bbcc4e0 100644 --- a/integrations/scraping-cuenca-cantabrico/src/integration.ts +++ b/integrations/scraping-cuenca-cantabrico/src/integration.ts @@ -1,6 +1,6 @@ -import { getCantabricoPayload } from "@/api"; -import { toRawRows } from "@/scraper/business"; -import { mapToEmbalseUpdateSAIH } from "@/scraper/mapper"; +import { getCantabricoPayload } from "./api/index.js"; +import { toRawRows } from "./scraper/business.js"; +import { mapToEmbalseUpdateSAIH } from "./scraper/mapper.js"; import type { EmbalseUpdateSAIHEntity } from "db-model"; /** Orquesta: API → normaliza → mapea a tu entidad final */ diff --git a/integrations/scraping-cuenca-cantabrico/src/scraper/business.ts b/integrations/scraping-cuenca-cantabrico/src/scraper/business.ts index d7c867a..d56eb4f 100644 --- a/integrations/scraping-cuenca-cantabrico/src/scraper/business.ts +++ b/integrations/scraping-cuenca-cantabrico/src/scraper/business.ts @@ -1,4 +1,4 @@ -import type { SaichFeature } from "@/api"; +import type { SaichFeature } from "../api/index.js"; export interface RawRow { id: number; // codigo_general diff --git a/integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts b/integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts index 861bbab..43ad26f 100644 --- a/integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts +++ b/integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts @@ -1,5 +1,5 @@ import type { EmbalseUpdateSAIHEntity } from "db-model"; -import type { RawRow } from "./business"; +import type { RawRow } from "./business.js"; /** Mapea a EmbalseUpdateSAIHEntity*/ export function mapToEmbalseUpdateSAIH( diff --git a/integrations/scraping-cuenca-cantabrico/tsconfig.json b/integrations/scraping-cuenca-cantabrico/tsconfig.json index 87d8cea..ce7b1d4 100644 --- a/integrations/scraping-cuenca-cantabrico/tsconfig.json +++ b/integrations/scraping-cuenca-cantabrico/tsconfig.json @@ -1,25 +1,16 @@ { - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "bundler", - "skipLibCheck": true, - "isolatedModules": true, - "esModuleInterop": true, - "baseUrl": "./", - "paths": { - "@/*": [ - "src/*" - ], - "@/api/*": [ - "src/api/*" - ], - "@/scraper/*": [ - "src/scraper/*" - ] - } - }, - "include": [ - "src" - ] -} \ No newline at end of file + "compilerOptions": { + "target": "ESNext", + "module": "nodenext", + "moduleResolution": "nodenext", + "outDir": "dist", + "skipLibCheck": true, + "isolatedModules": true, + "esModuleInterop": true, + "verbatimModuleSyntax": false, + "declaration": true, + "baseUrl": "./" + }, + "include": ["src/**/*"], + "exclude": ["dist", "node_modules"] +} diff --git a/integrations/scraping-cuenca-mediterranea/package.json b/integrations/scraping-cuenca-mediterranea/package.json index 7d26c06..acb0bcf 100644 --- a/integrations/scraping-cuenca-mediterranea/package.json +++ b/integrations/scraping-cuenca-mediterranea/package.json @@ -12,7 +12,7 @@ "start": "tsx --watch src/console-runner.ts", "build": "run-p clean type-check build:scraping-cuenca-mediterranea", "build:scraping-cuenca-mediterranea": "tsc", - "clean": "rimraf dist", + "clean": "rimraf dist", "type-check": "tsc --noEmit --preserveWatchOutput" }, "dependencies": { diff --git a/package-lock.json b/package-lock.json index ac12f28..2d905e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2129,6 +2129,7 @@ "version": "24.3.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.10.0" } @@ -2153,6 +2154,7 @@ "version": "19.1.10", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.0.2" } @@ -2952,6 +2954,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -4499,6 +4502,7 @@ "integrity": "sha512-QgODejq9K3OzoBbuyobZlUhznP5SKwPqp+6Q6xw6o8gnhr4O85L2U915iM2IDcfF2NPXVaM9zlo9tdwipnYwzg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -4657,6 +4661,7 @@ "node_modules/react": { "version": "19.1.1", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -4664,6 +4669,7 @@ "node_modules/react-dom": { "version": "19.1.1", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.26.0" }, @@ -5436,6 +5442,7 @@ "version": "5.9.2", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -5467,6 +5474,7 @@ "version": "7.1.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", @@ -5862,6 +5870,7 @@ "@types/prompts": "^2.4.9", "arcgis": "*", "prompts": "^2.4.2", + "scraping-cuenca-cantabrico": "*", "scraping-cuenca-mediterranea": "*" } }, diff --git a/packages/db/package.json b/packages/db/package.json index 5cd70a0..7e8bea5 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -28,6 +28,7 @@ "@types/prompts": "^2.4.9", "arcgis": "*", "prompts": "^2.4.2", - "scraping-cuenca-mediterranea": "*" + "scraping-cuenca-mediterranea": "*", + "scraping-cuenca-cantabrico": "*" } } diff --git a/packages/db/src/dals/embalses/embalses.mappers.ts b/packages/db/src/dals/embalses/embalses.mappers.ts index 1b0cd5a..6b915be 100644 --- a/packages/db/src/dals/embalses/embalses.mappers.ts +++ b/packages/db/src/dals/embalses/embalses.mappers.ts @@ -34,3 +34,23 @@ export const mapperFromCuencasMediterraneaToArcgis = new Map< [379, { nombre: "Jarrama", idArcgis: 358 }], [380, { nombre: "Andévalo", idArcgis: 355 }], ]); + +// Ojo Embalse de La Florida-Pilotuerto, Valdemurio, El Furacón, Priañes, Cordiñanes, La Lastra, Leurtza Inferior, Leurtza Superior, Endara-San Antón + +export const mapperFromCuencasCantabricoToArcgis = new Map< + number, + InfoDestinoArcgis +>([ + [1406, { nombre: "Salime", idArcgis: 270 }], + [1408, { nombre: "Doiras", idArcgis: 121 }], + [1409, { nombre: "Arbón", idArcgis: 26 }], + [1356, { nombre: "La Barca", idArcgis: 40 }], + [1333, { nombre: "Tanes", idArcgis: 305 }], + [1334, { nombre: "Rioseco", idArcgis: 385 }], + [1345, { nombre: "Alfilorios", idArcgis: 17 }], + [1253, { nombre: "La Cohilla", idArcgis: 103 }], + [1231, { nombre: "Alsa - Mediajo", idArcgis: 310 }], + [1177, { nombre: "Ordunte", idArcgis: 217 }], + [1078, { nombre: "Ibiur", idArcgis: 377 }], + [1108, { nombre: "Añarbe", idArcgis: 34 }], +]); diff --git a/packages/db/src/dals/embalses/embalses.repository.ts b/packages/db/src/dals/embalses/embalses.repository.ts index fef899f..ca7e5e3 100644 --- a/packages/db/src/dals/embalses/embalses.repository.ts +++ b/packages/db/src/dals/embalses/embalses.repository.ts @@ -1,7 +1,8 @@ import { scrapeSeedEmbalses } from "arcgis"; import { getEmbalsesContext } from "./embalses.context.js"; -import { mapperFromCuencasMediterraneaToArcgis } from "./embalses.mappers.js"; +import { mapperFromCuencasMediterraneaToArcgis, mapperFromCuencasCantabricoToArcgis } from "./embalses.mappers.js"; import { scrapeCuencaMediterranea } from "scraping-cuenca-mediterranea"; +import { scrapeCuencaCantabrica } from 'scraping-cuenca-cantabrico'; import { parseDate } from "./embalses.helpers.js"; export const embalsesRepository = { @@ -72,4 +73,57 @@ export const embalsesRepository = { return actualizados > 0; }, + actualizarCuencaCantabrico: async (): Promise => { + const embalsesCantabrica = await scrapeCuencaCantabrica(); + + console.log( + `Se han scrapeado ${embalsesCantabrica.length} embalses de la Cuenca Cantábrica` + ); + + let actualizados = 0; + let noEncontrados = 0; + let sinMapper = 0; + + for (const embalse of embalsesCantabrica) { + const infoDestino = mapperFromCuencasCantabricoToArcgis.get(embalse.id); + + if (!infoDestino) { + sinMapper++; + console.warn(`Sin mapper para ID ${embalse.id} - ${embalse.nombre}`); + continue; + } + + console.log( + `🔍 Mapeando: ID scraping ${embalse.id} -> _id BD ${infoDestino.idArcgis} (${infoDestino.nombre})` + ); + + const { matchedCount } = await getEmbalsesContext().updateOne( + { _id: infoDestino.idArcgis.toString() }, + { + $set: { + aguaActualSAIH: embalse.aguaActualSAIH, + fechaMedidaAguaActualSAIH: parseDate(embalse.fechaMedidaSAIH), + }, + } + ); + + if (matchedCount > 0) { + actualizados++; + console.log( + `Actualizado: ${infoDestino.nombre} (_id: ${infoDestino.idArcgis}) -> ${embalse.aguaActualSAIH} hm³` + ); + } else { + noEncontrados++; + console.warn( + `No encontrado en BD: _id ${infoDestino.idArcgis} - ${infoDestino.nombre}` + ); + } + } + + console.log( + `Resumen Cuenca Mediterránea: ${actualizados} actualizados, ${noEncontrados} no encontrados, ${sinMapper} sin mapper` + ); + + return actualizados > 0; + } }; From 178a1bd1cf474ed3e6c7605b54acd04ee181ca70 Mon Sep 17 00:00:00 2001 From: Antonio Contreras Date: Mon, 9 Feb 2026 12:57:01 +0100 Subject: [PATCH 2/4] fix some mistakes --- functions/src/functions/scraping-functions.ts | 4 ++-- packages/db/src/dals/embalses/embalses.mappers.ts | 2 +- packages/db/src/dals/embalses/embalses.repository.ts | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/functions/src/functions/scraping-functions.ts b/functions/src/functions/scraping-functions.ts index 98ca007..2c43e68 100644 --- a/functions/src/functions/scraping-functions.ts +++ b/functions/src/functions/scraping-functions.ts @@ -22,10 +22,10 @@ export async function scrapingsFunction( } if (responseCuencaCantabrico) { - context.log(`Se han actualizado los embalses de la cuenca Cantábrico`); + context.log(`Se han actualizado los embalses de la cuenca Cantábrica`); } else { context.log( - "No se han podido actualizar los embalses de la cuenca Cantábrico" + "No se han podido actualizar los embalses de la cuenca Cantábrica" ); } diff --git a/packages/db/src/dals/embalses/embalses.mappers.ts b/packages/db/src/dals/embalses/embalses.mappers.ts index 6b915be..59b501d 100644 --- a/packages/db/src/dals/embalses/embalses.mappers.ts +++ b/packages/db/src/dals/embalses/embalses.mappers.ts @@ -35,7 +35,7 @@ export const mapperFromCuencasMediterraneaToArcgis = new Map< [380, { nombre: "Andévalo", idArcgis: 355 }], ]); -// Ojo Embalse de La Florida-Pilotuerto, Valdemurio, El Furacón, Priañes, Cordiñanes, La Lastra, Leurtza Inferior, Leurtza Superior, Endara-San Antón +// Ojo Embalse de La Florida-Pilotuerto, Valdemurio, El Furacón, Priañes, Cordiñanes, La Lastra, Palombera, Arriarán, Leurtza Inferior, Leurtza Superior, Endara-San Antón export const mapperFromCuencasCantabricoToArcgis = new Map< number, diff --git a/packages/db/src/dals/embalses/embalses.repository.ts b/packages/db/src/dals/embalses/embalses.repository.ts index ca7e5e3..93c1b49 100644 --- a/packages/db/src/dals/embalses/embalses.repository.ts +++ b/packages/db/src/dals/embalses/embalses.repository.ts @@ -74,17 +74,17 @@ export const embalsesRepository = { return actualizados > 0; }, actualizarCuencaCantabrico: async (): Promise => { - const embalsesCantabrica = await scrapeCuencaCantabrica(); + const embalsesCantabrico = await scrapeCuencaCantabrica(); console.log( - `Se han scrapeado ${embalsesCantabrica.length} embalses de la Cuenca Cantábrica` + `Se han scrapeado ${embalsesCantabrico.length} embalses de la Cuenca Cantábrica` ); let actualizados = 0; let noEncontrados = 0; let sinMapper = 0; - for (const embalse of embalsesCantabrica) { + for (const embalse of embalsesCantabrico) { const infoDestino = mapperFromCuencasCantabricoToArcgis.get(embalse.id); if (!infoDestino) { @@ -121,7 +121,7 @@ export const embalsesRepository = { } console.log( - `Resumen Cuenca Mediterránea: ${actualizados} actualizados, ${noEncontrados} no encontrados, ${sinMapper} sin mapper` + `Resumen Cuenca Cantábrico: ${actualizados} actualizados, ${noEncontrados} no encontrados, ${sinMapper} sin mapper` ); return actualizados > 0; From e361a5fc58601dd29ac0e0982daa0141d308f4fa Mon Sep 17 00:00:00 2001 From: Antonio Contreras Date: Mon, 9 Feb 2026 16:53:45 +0100 Subject: [PATCH 3/4] fix embalses and other minor things --- integrations/arcgis/src/embalse-provincias.ts | 2 +- .../scraping-cuenca-cantabrico/src/scraper/helpers.ts | 5 +++++ .../scraping-cuenca-cantabrico/src/scraper/mapper.ts | 3 ++- packages/db/src/dals/embalses/embalses.mappers.ts | 4 +++- 4 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 integrations/scraping-cuenca-cantabrico/src/scraper/helpers.ts diff --git a/integrations/arcgis/src/embalse-provincias.ts b/integrations/arcgis/src/embalse-provincias.ts index 43ed5d9..d431d4a 100644 --- a/integrations/arcgis/src/embalse-provincias.ts +++ b/integrations/arcgis/src/embalse-provincias.ts @@ -312,7 +312,7 @@ export const embalseProvincias = new Map([ ["salime", "Asturias"], ["sallente", "Lleida"], ["salor", "Cáceres"], - ["san-anton", "Almería"], + ["san-anton", "Navarra"], ["san-bartolome", "Huelva"], ["san-clemente", "Huelva"], ["san-jose", "Almería"], diff --git a/integrations/scraping-cuenca-cantabrico/src/scraper/helpers.ts b/integrations/scraping-cuenca-cantabrico/src/scraper/helpers.ts new file mode 100644 index 0000000..829f3e4 --- /dev/null +++ b/integrations/scraping-cuenca-cantabrico/src/scraper/helpers.ts @@ -0,0 +1,5 @@ +export const mapStringToDateFormat = (strDate: string) => { + const [year, month, day] = strDate.split(" ")[0].split('-'); + + return `${day}/${month}/${year}`; +} diff --git a/integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts b/integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts index 43ad26f..a4addad 100644 --- a/integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts +++ b/integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts @@ -1,5 +1,6 @@ import type { EmbalseUpdateSAIHEntity } from "db-model"; import type { RawRow } from "./business.js"; +import { mapStringToDateFormat } from './helpers.js' /** Mapea a EmbalseUpdateSAIHEntity*/ export function mapToEmbalseUpdateSAIH( @@ -9,6 +10,6 @@ export function mapToEmbalseUpdateSAIH( id: r.id, nombre: r.nombre, aguaActualSAIH: r.volumenActualHm3, - fechaMedidaSAIH: r.fecha, + fechaMedidaSAIH: mapStringToDateFormat(r.fecha), })); } diff --git a/packages/db/src/dals/embalses/embalses.mappers.ts b/packages/db/src/dals/embalses/embalses.mappers.ts index 59b501d..663af45 100644 --- a/packages/db/src/dals/embalses/embalses.mappers.ts +++ b/packages/db/src/dals/embalses/embalses.mappers.ts @@ -35,7 +35,8 @@ export const mapperFromCuencasMediterraneaToArcgis = new Map< [380, { nombre: "Andévalo", idArcgis: 355 }], ]); -// Ojo Embalse de La Florida-Pilotuerto, Valdemurio, El Furacón, Priañes, Cordiñanes, La Lastra, Palombera, Arriarán, Leurtza Inferior, Leurtza Superior, Endara-San Antón +// Ojo Embalse de La Florida-Pilotuerto, Valdemurio, El Furacón, Priañes, Cordiñanes, La Lastra, Palombera, Arriarán, Leurtza Inferior, Leurtza Superior + export const mapperFromCuencasCantabricoToArcgis = new Map< number, @@ -53,4 +54,5 @@ export const mapperFromCuencasCantabricoToArcgis = new Map< [1177, { nombre: "Ordunte", idArcgis: 217 }], [1078, { nombre: "Ibiur", idArcgis: 377 }], [1108, { nombre: "Añarbe", idArcgis: 34 }], + [1847, { nombre: "San Antón", idArcgis: 272 }], ]); From 3f09351995c7bf56d411e36f6d2dfd32c089ecc1 Mon Sep 17 00:00:00 2001 From: Antonio Contreras Date: Tue, 10 Feb 2026 16:57:03 +0100 Subject: [PATCH 4/4] fix name to mapper --- .../scraping-cuenca-cantabrico/src/scraper/helpers.ts | 2 +- integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integrations/scraping-cuenca-cantabrico/src/scraper/helpers.ts b/integrations/scraping-cuenca-cantabrico/src/scraper/helpers.ts index 829f3e4..d92bf32 100644 --- a/integrations/scraping-cuenca-cantabrico/src/scraper/helpers.ts +++ b/integrations/scraping-cuenca-cantabrico/src/scraper/helpers.ts @@ -1,4 +1,4 @@ -export const mapStringToDateFormat = (strDate: string) => { +export const mapStringToApiDate = (strDate: string) => { const [year, month, day] = strDate.split(" ")[0].split('-'); return `${day}/${month}/${year}`; diff --git a/integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts b/integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts index a4addad..b6689ef 100644 --- a/integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts +++ b/integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts @@ -1,6 +1,6 @@ import type { EmbalseUpdateSAIHEntity } from "db-model"; import type { RawRow } from "./business.js"; -import { mapStringToDateFormat } from './helpers.js' +import { mapStringToApiDate } from './helpers.js' /** Mapea a EmbalseUpdateSAIHEntity*/ export function mapToEmbalseUpdateSAIH( @@ -10,6 +10,6 @@ export function mapToEmbalseUpdateSAIH( id: r.id, nombre: r.nombre, aguaActualSAIH: r.volumenActualHm3, - fechaMedidaSAIH: mapStringToDateFormat(r.fecha), + fechaMedidaSAIH: mapStringToApiDate(r.fecha), })); }