diff --git a/functions/src/functions/scraping-functions.ts b/functions/src/functions/scraping-functions.ts index 7ab68f4..8915603 100644 --- a/functions/src/functions/scraping-functions.ts +++ b/functions/src/functions/scraping-functions.ts @@ -11,6 +11,8 @@ export async function scrapingsFunction( const responseCuencaMediterranea = await embalsesRepository.actualizarCuencaMediterranea(); + const responseCuencaCatalana = await embalsesRepository.actualizarCuencaCatalana(); + if (responseCuencaMediterranea) { context.log(`Se han actualizado los embalses de la cuenca Mediterránea`); } else { @@ -18,6 +20,14 @@ export async function scrapingsFunction( "No se han podido actualizar los embalses de la cuenca Mediterránea" ); } + + if (responseCuencaCatalana) { + context.log(`Se han actualizado los embalses de la cuenca Catalana`); + } else { + context.log( + "No se han podido actualizar los embalses de la cuenca Catalana" + ); + } await dbServer.disconnect(); } diff --git a/integrations/scraping-cuenca-catalana/package.json b/integrations/scraping-cuenca-catalana/package.json index 88cee44..c31bb24 100644 --- a/integrations/scraping-cuenca-catalana/package.json +++ b/integrations/scraping-cuenca-catalana/package.json @@ -4,10 +4,16 @@ "private": "true", "type": "module", "exports": { - ".": "./src/index.js" + ".": "./dist/index.js" }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", "scripts": { - "start": "tsx --watch src/console-runner.ts" + "start": "tsx --watch src/console-runner.ts", + "build": "run-p clean type-check build:scraping-cuenca-catalana", + "build:scraping-cuenca-catalana": "tsc", + "clean": "rimraf dist", + "type-check": "tsc --noEmit --preserveWatchOutput" }, "dependencies": { "db-model": "^1.0.0" diff --git a/integrations/scraping-cuenca-catalana/src/api/cuenca.api.ts b/integrations/scraping-cuenca-catalana/src/api/cuenca.api.ts index d435b52..2a1eda3 100644 --- a/integrations/scraping-cuenca-catalana/src/api/cuenca.api.ts +++ b/integrations/scraping-cuenca-catalana/src/api/cuenca.api.ts @@ -1,6 +1,6 @@ import axios from 'axios'; -import { EmbalseCatalanApi } from './cuenca.api-model'; -import { mapApiToEmbalses } from '../cuenca.mapper'; +import { EmbalseCatalanApi } from './cuenca.api-model.js'; +import { mapApiToEmbalses } from '../cuenca.mapper.js'; import { EmbalseUpdateSAIHEntity } from 'db-model'; /** diff --git a/integrations/scraping-cuenca-catalana/src/api/index.ts b/integrations/scraping-cuenca-catalana/src/api/index.ts index 13c7ed6..62c405f 100644 --- a/integrations/scraping-cuenca-catalana/src/api/index.ts +++ b/integrations/scraping-cuenca-catalana/src/api/index.ts @@ -1 +1 @@ -export * from "./cuenca.api"; +export * from "./cuenca.api.js"; diff --git a/integrations/scraping-cuenca-catalana/src/console-runner.ts b/integrations/scraping-cuenca-catalana/src/console-runner.ts index 1f56a01..db75010 100644 --- a/integrations/scraping-cuenca-catalana/src/console-runner.ts +++ b/integrations/scraping-cuenca-catalana/src/console-runner.ts @@ -1,6 +1,5 @@ -import { integracionCuencaCatalana } from "./integrations"; -import { URL } from "./integrations"; +import { integracionCuencaCatalana } from "./integrations.js"; console.log("Estados de las Cuencas Catalanas:"); -const result = await integracionCuencaCatalana(URL); +const result = await integracionCuencaCatalana(); console.log(JSON.stringify(result, null, 2)); diff --git a/integrations/scraping-cuenca-catalana/src/cuenca.mapper.ts b/integrations/scraping-cuenca-catalana/src/cuenca.mapper.ts index 2954984..551075a 100644 --- a/integrations/scraping-cuenca-catalana/src/cuenca.mapper.ts +++ b/integrations/scraping-cuenca-catalana/src/cuenca.mapper.ts @@ -1,6 +1,6 @@ -import { EmbalseCatalanApi } from './api/cuenca.api-model'; +import { EmbalseCatalanApi } from './api/cuenca.api-model.js'; import { EmbalseUpdateSAIHEntity } from 'db-model'; -import { formatApiDate, formatVolumeToFixedTwo } from './business'; +import { formatApiDate, formatVolumeToFixedTwo } from './business.js'; export function mapApiToEmbalses( apiData: Record diff --git a/integrations/scraping-cuenca-catalana/src/index.ts b/integrations/scraping-cuenca-catalana/src/index.ts index 5eb5b72..5bbff7b 100644 --- a/integrations/scraping-cuenca-catalana/src/index.ts +++ b/integrations/scraping-cuenca-catalana/src/index.ts @@ -1 +1 @@ -export * from "./integrations"; +export * from "./integrations.js"; diff --git a/integrations/scraping-cuenca-catalana/src/integrations.ts b/integrations/scraping-cuenca-catalana/src/integrations.ts index 5c3c62f..a1778b7 100644 --- a/integrations/scraping-cuenca-catalana/src/integrations.ts +++ b/integrations/scraping-cuenca-catalana/src/integrations.ts @@ -1,10 +1,10 @@ import { EmbalseUpdateSAIHEntity } from 'db-model'; -import { getCuencaCatalana } from './api'; +import { getCuencaCatalana } from './api/cuenca.api.js'; -export const URL = +const URL = 'https://aplicacions.aca.gencat.cat/aetr/vishid/v2/data/public/reservoir/capacity'; -export async function integracionCuencaCatalana(URL: string) { +export async function integracionCuencaCatalana() { const embalses: EmbalseUpdateSAIHEntity[] = await getCuencaCatalana(URL); return embalses; } diff --git a/integrations/scraping-cuenca-catalana/tsconfig.json b/integrations/scraping-cuenca-catalana/tsconfig.json index fad30e0..ce7b1d4 100644 --- a/integrations/scraping-cuenca-catalana/tsconfig.json +++ b/integrations/scraping-cuenca-catalana/tsconfig.json @@ -1,17 +1,16 @@ { "compilerOptions": { "target": "ESNext", - "module": "ESNext", - "moduleResolution": "bundler", + "module": "nodenext", + "moduleResolution": "nodenext", + "outDir": "dist", "skipLibCheck": true, "isolatedModules": true, "esModuleInterop": true, - "baseUrl": "./", - "paths": { - "@/*": ["src/*"], - "@/api/*": ["src/api/*"], - "@/scraper/*": ["src/scraper/*"] - } + "verbatimModuleSyntax": false, + "declaration": true, + "baseUrl": "./" }, - "include": ["src"] + "include": ["src/**/*"], + "exclude": ["dist", "node_modules"] } diff --git a/packages/db/package.json b/packages/db/package.json index 5cd70a0..3e0b370 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -28,6 +28,7 @@ "@types/prompts": "^2.4.9", "arcgis": "*", "prompts": "^2.4.2", - "scraping-cuenca-mediterranea": "*" + "scraping-cuenca-mediterranea": "*", + "scraping-cuenca-catalana": "*" } } diff --git a/packages/db/src/dals/embalses/embalses.mappers.ts b/packages/db/src/dals/embalses/embalses.mappers.ts index 1b0cd5a..88090e4 100644 --- a/packages/db/src/dals/embalses/embalses.mappers.ts +++ b/packages/db/src/dals/embalses/embalses.mappers.ts @@ -34,3 +34,20 @@ export const mapperFromCuencasMediterraneaToArcgis = new Map< [379, { nombre: "Jarrama", idArcgis: 358 }], [380, { nombre: "Andévalo", idArcgis: 355 }], ]); + +// OJO Pasteral, Gaià, Siurana, Foix + +// id => 89 Está en Lleida pero pertenece a la cuenca del Ebro. + +export const mapperFromCuencasCatalanaToArcgis = new Map< + number, + InfoDestinoArcgis +>([ + [171169001, { nombre: "Susqueda", idArcgis: 301 }], + [170600001, { nombre: "Boadella", idArcgis: 53 }], + [430537001, { nombre: "Riudecanyes", idArcgis: 259 }], + [83036001, { nombre: "Sau", idArcgis: 290 }], + [81419003, { nombre: "La Llosa del Cavall", idArcgis: 178 }], + [82687001, { nombre: "La Baells", idArcgis: 36 }], + [250753004, { nombre: "Sant Pons", idArcgis: 280 }], +]); diff --git a/packages/db/src/dals/embalses/embalses.repository.ts b/packages/db/src/dals/embalses/embalses.repository.ts index fef899f..c38c5f9 100644 --- a/packages/db/src/dals/embalses/embalses.repository.ts +++ b/packages/db/src/dals/embalses/embalses.repository.ts @@ -1,7 +1,8 @@ import { scrapeSeedEmbalses } from "arcgis"; import { getEmbalsesContext } from "./embalses.context.js"; -import { mapperFromCuencasMediterraneaToArcgis } from "./embalses.mappers.js"; +import { mapperFromCuencasMediterraneaToArcgis, mapperFromCuencasCatalanaToArcgis } from "./embalses.mappers.js"; import { scrapeCuencaMediterranea } from "scraping-cuenca-mediterranea"; +import { integracionCuencaCatalana } from 'scraping-cuenca-catalana'; import { parseDate } from "./embalses.helpers.js"; export const embalsesRepository = { @@ -72,4 +73,57 @@ export const embalsesRepository = { return actualizados > 0; }, + actualizarCuencaCatalana: async (): Promise => { + const embalsesCatalana = await integracionCuencaCatalana(); + + console.log( + `Se han scrapeado ${embalsesCatalana.length} embalses de la Cuenca Catalana` + ); + + let actualizados = 0; + let noEncontrados = 0; + let sinMapper = 0; + + for (const embalse of embalsesCatalana) { + const infoDestino = mapperFromCuencasCatalanaToArcgis.get(embalse.id); + + if (!infoDestino) { + sinMapper++; + console.warn(`Sin mapper para ID ${embalse.id} - ${embalse.nombre}`); + continue; + } + + console.log( + `🔍 Mapeando: ID scraping ${embalse.id} -> _id BD ${infoDestino.idArcgis} (${infoDestino.nombre})` + ); + + const { matchedCount } = await getEmbalsesContext().updateOne( + { _id: infoDestino.idArcgis.toString() }, + { + $set: { + aguaActualSAIH: embalse.aguaActualSAIH, + fechaMedidaAguaActualSAIH: parseDate(embalse.fechaMedidaSAIH), + }, + } + ); + + if (matchedCount > 0) { + actualizados++; + console.log( + `Actualizado: ${infoDestino.nombre} (_id: ${infoDestino.idArcgis}) -> ${embalse.aguaActualSAIH} hm³` + ); + } else { + noEncontrados++; + console.warn( + `No encontrado en BD: _id ${infoDestino.idArcgis} - ${infoDestino.nombre}` + ); + } + } + + console.log( + `Resumen Cuenca Catalana: ${actualizados} actualizados, ${noEncontrados} no encontrados, ${sinMapper} sin mapper` + ); + + return actualizados > 0; + } };