Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions functions/src/functions/scraping-functions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,24 @@ export async function scrapingsFunction(
const responseCuencaMediterranea =
await embalsesRepository.actualizarCuencaMediterranea();

const responseCuencaCantabrico = await embalsesRepository.actualizarCuencaCantabrico();

if (responseCuencaMediterranea) {
context.log(`Se han actualizado los embalses de la cuenca Mediterránea`);
} else {
context.log(
"No se han podido actualizar los embalses de la cuenca Mediterránea"
);
}

if (responseCuencaCantabrico) {
context.log(`Se han actualizado los embalses de la cuenca Cantábrica`);
} else {
context.log(
"No se han podido actualizar los embalses de la cuenca Cantábrica"
);
}

await dbServer.disconnect();
}

Expand Down
2 changes: 1 addition & 1 deletion integrations/arcgis/src/embalse-provincias.ts
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ export const embalseProvincias = new Map<string, string>([
["salime", "Asturias"],
["sallente", "Lleida"],
["salor", "Cáceres"],
["san-anton", "Almería"],
["san-anton", "Navarra"],
["san-bartolome", "Huelva"],
["san-clemente", "Huelva"],
["san-jose", "Almería"],
Expand Down
12 changes: 9 additions & 3 deletions integrations/scraping-cuenca-cantabrico/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,18 @@
"private": true,
"type": "module",
"exports": {
".": "./src/index.ts"
".": "./dist/index.js"
},
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"scripts": {
"start": "tsx --watch src/console-runner.ts"
"start": "tsx --watch src/console-runner.ts",
"build": "run-p clean type-check build:scraping-cuenca-cantabrico",
"build:scraping-cuenca-cantabrico": "tsc",
"clean": "rimraf dist",
"type-check": "tsc --noEmit --preserveWatchOutput"
},
"dependencies": {
"db-model": "^1.0.0"
}
}
}
1 change: 1 addition & 0 deletions integrations/scraping-cuenca-cantabrico/src/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
export * from "./integration.js";
6 changes: 3 additions & 3 deletions integrations/scraping-cuenca-cantabrico/src/integration.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { getCantabricoPayload } from "@/api";
import { toRawRows } from "@/scraper/business";
import { mapToEmbalseUpdateSAIH } from "@/scraper/mapper";
import { getCantabricoPayload } from "./api/index.js";
import { toRawRows } from "./scraper/business.js";
import { mapToEmbalseUpdateSAIH } from "./scraper/mapper.js";
import type { EmbalseUpdateSAIHEntity } from "db-model";

/** Orquesta: API → normaliza → mapea a tu entidad final */
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { SaichFeature } from "@/api";
import type { SaichFeature } from "../api/index.js";

export interface RawRow {
id: number; // codigo_general
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
export const mapStringToApiDate = (strDate: string) => {
const [year, month, day] = strDate.split(" ")[0].split('-');

return `${day}/${month}/${year}`;
}
5 changes: 3 additions & 2 deletions integrations/scraping-cuenca-cantabrico/src/scraper/mapper.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { EmbalseUpdateSAIHEntity } from "db-model";
import type { RawRow } from "./business";
import type { RawRow } from "./business.js";
import { mapStringToApiDate } from './helpers.js'

/** Mapea a EmbalseUpdateSAIHEntity*/
export function mapToEmbalseUpdateSAIH(
Expand All @@ -9,6 +10,6 @@ export function mapToEmbalseUpdateSAIH(
id: r.id,
nombre: r.nombre,
aguaActualSAIH: r.volumenActualHm3,
fechaMedidaSAIH: r.fecha,
fechaMedidaSAIH: mapStringToApiDate(r.fecha),
}));
}
39 changes: 15 additions & 24 deletions integrations/scraping-cuenca-cantabrico/tsconfig.json
Original file line number Diff line number Diff line change
@@ -1,25 +1,16 @@
{
"compilerOptions": {
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "bundler",
"skipLibCheck": true,
"isolatedModules": true,
"esModuleInterop": true,
"baseUrl": "./",
"paths": {
"@/*": [
"src/*"
],
"@/api/*": [
"src/api/*"
],
"@/scraper/*": [
"src/scraper/*"
]
}
},
"include": [
"src"
]
}
"compilerOptions": {
"target": "ESNext",
"module": "nodenext",
"moduleResolution": "nodenext",
"outDir": "dist",
"skipLibCheck": true,
"isolatedModules": true,
"esModuleInterop": true,
"verbatimModuleSyntax": false,
"declaration": true,
"baseUrl": "./"
},
"include": ["src/**/*"],
"exclude": ["dist", "node_modules"]
}
2 changes: 1 addition & 1 deletion integrations/scraping-cuenca-mediterranea/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"start": "tsx --watch src/console-runner.ts",
"build": "run-p clean type-check build:scraping-cuenca-mediterranea",
"build:scraping-cuenca-mediterranea": "tsc",
"clean": "rimraf dist",
"clean": "rimraf dist",
"type-check": "tsc --noEmit --preserveWatchOutput"
},
"dependencies": {
Expand Down
9 changes: 9 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion packages/db/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
"@types/prompts": "^2.4.9",
"arcgis": "*",
"prompts": "^2.4.2",
"scraping-cuenca-mediterranea": "*"
"scraping-cuenca-mediterranea": "*",
"scraping-cuenca-cantabrico": "*"
}
}
22 changes: 22 additions & 0 deletions packages/db/src/dals/embalses/embalses.mappers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,25 @@ export const mapperFromCuencasMediterraneaToArcgis = new Map<
[379, { nombre: "Jarrama", idArcgis: 358 }],
[380, { nombre: "Andévalo", idArcgis: 355 }],
]);

// Ojo Embalse de La Florida-Pilotuerto, Valdemurio, El Furacón, Priañes, Cordiñanes, La Lastra, Palombera, Arriarán, Leurtza Inferior, Leurtza Superior


export const mapperFromCuencasCantabricoToArcgis = new Map<
number,
InfoDestinoArcgis
>([
[1406, { nombre: "Salime", idArcgis: 270 }],
[1408, { nombre: "Doiras", idArcgis: 121 }],
[1409, { nombre: "Arbón", idArcgis: 26 }],
[1356, { nombre: "La Barca", idArcgis: 40 }],
[1333, { nombre: "Tanes", idArcgis: 305 }],
[1334, { nombre: "Rioseco", idArcgis: 385 }],
[1345, { nombre: "Alfilorios", idArcgis: 17 }],
[1253, { nombre: "La Cohilla", idArcgis: 103 }],
[1231, { nombre: "Alsa - Mediajo", idArcgis: 310 }],
[1177, { nombre: "Ordunte", idArcgis: 217 }],
[1078, { nombre: "Ibiur", idArcgis: 377 }],
[1108, { nombre: "Añarbe", idArcgis: 34 }],
[1847, { nombre: "San Antón", idArcgis: 272 }],
]);
56 changes: 55 additions & 1 deletion packages/db/src/dals/embalses/embalses.repository.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { scrapeSeedEmbalses } from "arcgis";
import { getEmbalsesContext } from "./embalses.context.js";
import { mapperFromCuencasMediterraneaToArcgis } from "./embalses.mappers.js";
import { mapperFromCuencasMediterraneaToArcgis, mapperFromCuencasCantabricoToArcgis } from "./embalses.mappers.js";
import { scrapeCuencaMediterranea } from "scraping-cuenca-mediterranea";
import { scrapeCuencaCantabrica } from 'scraping-cuenca-cantabrico';
import { parseDate } from "./embalses.helpers.js";

export const embalsesRepository = {
Expand Down Expand Up @@ -72,4 +73,57 @@ export const embalsesRepository = {

return actualizados > 0;
},
actualizarCuencaCantabrico: async (): Promise<boolean> => {
const embalsesCantabrico = await scrapeCuencaCantabrica();

console.log(
`Se han scrapeado ${embalsesCantabrico.length} embalses de la Cuenca Cantábrica`
);

let actualizados = 0;
let noEncontrados = 0;
let sinMapper = 0;

for (const embalse of embalsesCantabrico) {
const infoDestino = mapperFromCuencasCantabricoToArcgis.get(embalse.id);

if (!infoDestino) {
sinMapper++;
console.warn(`Sin mapper para ID ${embalse.id} - ${embalse.nombre}`);
continue;
}

console.log(
`🔍 Mapeando: ID scraping ${embalse.id} -> _id BD ${infoDestino.idArcgis} (${infoDestino.nombre})`
);

const { matchedCount } = await getEmbalsesContext().updateOne(
{ _id: infoDestino.idArcgis.toString() },
{
$set: {
aguaActualSAIH: embalse.aguaActualSAIH,
fechaMedidaAguaActualSAIH: parseDate(embalse.fechaMedidaSAIH),
},
}
);

if (matchedCount > 0) {
actualizados++;
console.log(
`Actualizado: ${infoDestino.nombre} (_id: ${infoDestino.idArcgis}) -> ${embalse.aguaActualSAIH} hm³`
);
} else {
noEncontrados++;
console.warn(
`No encontrado en BD: _id ${infoDestino.idArcgis} - ${infoDestino.nombre}`
);
}
}

console.log(
`Resumen Cuenca Cantábrico: ${actualizados} actualizados, ${noEncontrados} no encontrados, ${sinMapper} sin mapper`
);

return actualizados > 0;
}
};