diff --git a/functions/src/functions/scraping-functions.ts b/functions/src/functions/scraping-functions.ts index 153a790..a25abb8 100644 --- a/functions/src/functions/scraping-functions.ts +++ b/functions/src/functions/scraping-functions.ts @@ -29,6 +29,8 @@ export async function scrapingsFunction( const responseCuencaDuero = await embalsesRepository.actualizarCuencaDuero(); + const responseCuencaGuadalquivir = await embalsesRepository.actualizarCuencaGuadalquivir(); + if (responseCuencaMediterranea) { context.log( "scrapings-function: Se han actualizado los embalses de la cuenca Mediterránea", @@ -61,6 +63,14 @@ export async function scrapingsFunction( context.log( "No se han podido actualizar los embalses de la cuenca Duero") } + + if (responseCuencaGuadalquivir) { + context.log(`Se han actualizado los embalses de la cuenca Guadalquivir`); + } else { + context.log( + "No se han podido actualizar los embalses de la cuenca Guadalquivir") + } + } catch (error) { context.error("scrapings-function: ERROR", error); throw error; diff --git a/integrations/scraping-cuenca-guadalquivir/package.json b/integrations/scraping-cuenca-guadalquivir/package.json index 91df594..8201af1 100644 --- a/integrations/scraping-cuenca-guadalquivir/package.json +++ b/integrations/scraping-cuenca-guadalquivir/package.json @@ -4,10 +4,16 @@ "private": true, "type": "module", "exports": { - ".": "./src/index.ts" + ".": "./dist/index.js" }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", "scripts": { - "start": "tsx --watch src/console-runner.ts" + "start": "tsx --watch src/console-runner.ts", + "build": "run-p clean type-check build:scraping-cuenca-guadalquivir", + "build:scraping-cuenca-guadalquivir": "tsc", + "clean": "rimraf dist", + "type-check": "tsc --noEmit --preserveWatchOutput" }, "dependencies": { "db-model": "^1.0.0" diff --git a/integrations/scraping-cuenca-guadalquivir/src/api/index.ts b/integrations/scraping-cuenca-guadalquivir/src/api/index.ts index 8fcffea..b9296d5 100644 --- a/integrations/scraping-cuenca-guadalquivir/src/api/index.ts +++ b/integrations/scraping-cuenca-guadalquivir/src/api/index.ts @@ -1,2 +1,2 @@ -export * from "./cuenca.api"; -export * from "./cuenca.model"; +export * from "./cuenca.api.js"; +export * from "./cuenca.model.js"; diff --git a/integrations/scraping-cuenca-guadalquivir/src/console-runner.ts b/integrations/scraping-cuenca-guadalquivir/src/console-runner.ts index 121ccce..daca9fe 100644 --- a/integrations/scraping-cuenca-guadalquivir/src/console-runner.ts +++ b/integrations/scraping-cuenca-guadalquivir/src/console-runner.ts @@ -1,8 +1,6 @@ -import { scrapeCuencaGuadalquivir } from "./integration"; - -const url = "https://www.chguadalquivir.es/saih/"; +import { scrapeCuencaGuadalquivir } from "./integration.js"; console.log("Estado de la Cuenca del Guadalquivir"); -const result = await scrapeCuencaGuadalquivir(url); +const result = await scrapeCuencaGuadalquivir(); console.log(JSON.stringify(result, null, 2)); diff --git a/integrations/scraping-cuenca-guadalquivir/src/index.ts b/integrations/scraping-cuenca-guadalquivir/src/index.ts index 852c5e3..ba7a4ee 100644 --- a/integrations/scraping-cuenca-guadalquivir/src/index.ts +++ b/integrations/scraping-cuenca-guadalquivir/src/index.ts @@ -1 +1 @@ -export * from "./integration"; +export * from "./integration.js"; diff --git a/integrations/scraping-cuenca-guadalquivir/src/integration.ts b/integrations/scraping-cuenca-guadalquivir/src/integration.ts index 39417da..51af85f 100644 --- a/integrations/scraping-cuenca-guadalquivir/src/integration.ts +++ b/integrations/scraping-cuenca-guadalquivir/src/integration.ts @@ -1,12 +1,14 @@ -import { getCuencaPageContent, Zone, ZoneInfo, ZONES } from "./api"; +import { getCuencaPageContent, Zone, ZoneInfo, ZONES } from "./api/index.js"; import { extractCurrentDate, mapToEmbalsesByZone, mapToEmbalseUpdateSAIH, reservoirInfoFromTable, -} from "./scraper"; +} from "./scraper/index.js"; import { Browser, Page } from "playwright"; +const url = "https://www.chguadalquivir.es/saih/"; + async function processZoneData(page: Page, zone: Zone): Promise { const rawReservoirs = await reservoirInfoFromTable(page); const currentDate = await extractCurrentDate(page); @@ -14,9 +16,7 @@ async function processZoneData(page: Page, zone: Zone): Promise { return mapToEmbalsesByZone(zone.codigo, zone.nombre, saihReservoirs); } -export const scrapeCuencaGuadalquivir = async ( - url: string -): Promise => { +export const scrapeCuencaGuadalquivir = async (): Promise => { const reservoirsCollection: ZoneInfo[] = []; const zonesPromises = ZONES.map(async (zone) => { diff --git a/integrations/scraping-cuenca-guadalquivir/src/scraper/business.ts b/integrations/scraping-cuenca-guadalquivir/src/scraper/business.ts index 39de6c7..99658f8 100644 --- a/integrations/scraping-cuenca-guadalquivir/src/scraper/business.ts +++ b/integrations/scraping-cuenca-guadalquivir/src/scraper/business.ts @@ -1,4 +1,4 @@ -import { EmbalsesGuadalquivir } from "@/api"; +import { EmbalsesGuadalquivir } from "../api/index.js"; import { Locator, Page } from "playwright"; /** diff --git a/integrations/scraping-cuenca-guadalquivir/src/scraper/index.ts b/integrations/scraping-cuenca-guadalquivir/src/scraper/index.ts index 3ab44f4..db5a5fc 100644 --- a/integrations/scraping-cuenca-guadalquivir/src/scraper/index.ts +++ b/integrations/scraping-cuenca-guadalquivir/src/scraper/index.ts @@ -1,2 +1,2 @@ -export * from "./business"; -export * from "./mapper"; +export * from "./business.js"; +export * from "./mapper.js"; diff --git a/integrations/scraping-cuenca-guadalquivir/src/scraper/mapper.ts b/integrations/scraping-cuenca-guadalquivir/src/scraper/mapper.ts index afc9ea1..95d66c6 100644 --- a/integrations/scraping-cuenca-guadalquivir/src/scraper/mapper.ts +++ b/integrations/scraping-cuenca-guadalquivir/src/scraper/mapper.ts @@ -1,4 +1,4 @@ -import { EmbalsesGuadalquivir, ZoneInfo } from "@/api"; +import { EmbalsesGuadalquivir, ZoneInfo } from "../api/index.js"; import { EmbalseUpdateSAIHEntity } from "db-model"; export function mapToEmbalseUpdateSAIH( diff --git a/integrations/scraping-cuenca-guadalquivir/tsconfig.json b/integrations/scraping-cuenca-guadalquivir/tsconfig.json index fad30e0..ce7b1d4 100644 --- a/integrations/scraping-cuenca-guadalquivir/tsconfig.json +++ b/integrations/scraping-cuenca-guadalquivir/tsconfig.json @@ -1,17 +1,16 @@ { "compilerOptions": { "target": "ESNext", - "module": "ESNext", - "moduleResolution": "bundler", + "module": "nodenext", + "moduleResolution": "nodenext", + "outDir": "dist", "skipLibCheck": true, "isolatedModules": true, "esModuleInterop": true, - "baseUrl": "./", - "paths": { - "@/*": ["src/*"], - "@/api/*": ["src/api/*"], - "@/scraper/*": ["src/scraper/*"] - } + "verbatimModuleSyntax": false, + "declaration": true, + "baseUrl": "./" }, - "include": ["src"] + "include": ["src/**/*"], + "exclude": ["dist", "node_modules"] } diff --git a/packages/db/package.json b/packages/db/package.json index e50e6af..171dbe8 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -33,6 +33,7 @@ "scraping-cuenca-mediterranea": "*", "scraping-cuenca-cantabrico": "*", "scraping-cuenca-catalana": "*", - "scraping-cuenca-duero": "*" + "scraping-cuenca-duero": "*", + "scraping-cuenca-guadalquivir": "*" } } diff --git a/packages/db/src/dals/embalses/embalses.mappers.ts b/packages/db/src/dals/embalses/embalses.mappers.ts index 3077cc5..de0f01f 100644 --- a/packages/db/src/dals/embalses/embalses.mappers.ts +++ b/packages/db/src/dals/embalses/embalses.mappers.ts @@ -99,3 +99,76 @@ export const mapperFromCuencasDueroToArcgis = new Map< [31, { nombre: "Águeda", idArcgis: 5 }], [87, { nombre: "Castro", idArcgis: 26 }], ]) + +// OJO Guadanuño, Hornachuelos, Derivación Retortillo, Dañador, Encinarejo, El Infierno, El Renegado, Balsa Las Adelfas Contraemb. Bermejales, La Torre del Águila + +export const mapperFromCuencasGuadalquivirToArcgis = new Map< + number, + InfoDestinoArcgis +>([ + [26, { nombre: "Yeguas", idArcgis: 348 }], + [27, { nombre: "Martín Gonzalo", idArcgis: 196 }], + [28, { nombre: "Arenoso", idArcgis: 373 }], + [29, { nombre: "Guadalmellato", idArcgis: 155 }], + [30, { nombre: "San Rafael Navallana", idArcgis: 267 }], + [32, { nombre: "Vadomojón", idArcgis: 327 }], + [33, { nombre: "Sierra Boyera", idArcgis: 295 }], + [34, { nombre: "Puente Nuevo", idArcgis: 281 }], + [36, { nombre: "La Breña II", idArcgis: 376 }], + [37, { nombre: "Bembézar", idArcgis: 48 }], + [39, { nombre: "Retortillo", idArcgis: 253 }], + [48, { nombre: "Iznájar", idArcgis: 171 }], + [3, { nombre: "San Clemente", idArcgis: 274 }], + [4, { nombre: "El Portillo", idArcgis: 235 }], + [5, { nombre: "La Bolera", idArcgis: 55 }], + [6, { nombre: "Negratín", idArcgis: 211 }], + [7, { nombre: "Francisco Abellán", idArcgis: 134 }], + [41, { nombre: "Canales", idArcgis: 75 }], + [42, { nombre: "Quéntar", idArcgis: 247 }], + [44, { nombre: "Colomera", idArcgis: 104 }], + [45, { nombre: "Cubillas", idArcgis: 115 }], + [46, { nombre: "Los Bermejales", idArcgis: 52 }], + [1, { nombre: "El Tranco de Beas", idArcgis: 316 }], + [2, { nombre: "Aguascebas", idArcgis: 4 }], + [8, { nombre: "Siles", idArcgis: 383 }], + [13, { nombre: "Guadalén", idArcgis: 157 }], + [14, { nombre: "La Fernandina", idArcgis: 135 }], + [15, { nombre: "Guadalmena", idArcgis: 156 }], + [16, { nombre: "Giribaile", idArcgis: 148 }], + [17, { nombre: "Quiebrajano", idArcgis: 248 }], + [19, { nombre: "Rumblar", idArcgis: 264 }], + [21, { nombre: "Fresneda", idArcgis: 141 }], + [22, { nombre: "Montoro III", idArcgis: 207 }], + [23, { nombre: "Jándula", idArcgis: 175 }], + [31, { nombre: "Víboras", idArcgis: 359 }], + [54, { nombre: "José Torán", idArcgis: 174 }], + [55, { nombre: "La Puebla de Cazalla", idArcgis: 241 }], + [56, { nombre: "Huesna", idArcgis: 166 }], + [57, { nombre: "El Pintado", idArcgis: 231 }], + [58, { nombre: "Los Melonares", idArcgis: 382 }], + [61, { nombre: "Aracena", idArcgis: 25 }], + [62, { nombre: "Zufre", idArcgis: 351 }], + [63, { nombre: "La Minilla", idArcgis: 201 }], + [64, { nombre: "Cala", idArcgis: 69 }], + [65, { nombre: "Gergal", idArcgis: 147 }], + [67, { nombre: "El Agrio", idArcgis: 3 }], + [68, { nombre: "La Torre del Águila", idArcgis: 312 }], + [1, { nombre: "El Tranco de Beas", idArcgis: 316 }], + [4, { nombre: "El Portillo", idArcgis: 235 }], + [6, { nombre: "Negratín", idArcgis: 211 }], + [8, { nombre: "Siles", idArcgis: 383 }], + [13, { nombre: "Guadalén", idArcgis: 157 }], + [14, { nombre: "La Fernandina", idArcgis: 135 }], + [15, { nombre: "Guadalmena", idArcgis: 156 }], + [16, { nombre: "Giribaile", idArcgis: 148 }], + [23, { nombre: "Jándula", idArcgis: 175 }], + [26, { nombre: "Yeguas", idArcgis: 348 }], + [28, { nombre: "Arenoso", idArcgis: 373 }], + [30, { nombre: "San Rafael Navallana", idArcgis: 267 }], + [32, { nombre: "Vadomojón", idArcgis: 327 }], + [34, { nombre: "Puente Nuevo", idArcgis: 242 }], + [36, { nombre: "La Breña II", idArcgis: 376 }], + [48, { nombre: "Iznájar", idArcgis: 171 }], + [54, { nombre: "José Torán", idArcgis: 174 }], + [55, { nombre: "La Puebla de Cazalla", idArcgis: 241 }], +]) diff --git a/packages/db/src/dals/embalses/embalses.repository.ts b/packages/db/src/dals/embalses/embalses.repository.ts index 02090da..07eeed5 100644 --- a/packages/db/src/dals/embalses/embalses.repository.ts +++ b/packages/db/src/dals/embalses/embalses.repository.ts @@ -1,10 +1,11 @@ import { scrapeSeedEmbalses } from "arcgis"; import { getEmbalsesContext } from "./embalses.context.js"; -import { mapperFromCuencasMediterraneaToArcgis, mapperFromCuencasCantabricoToArcgis, mapperFromCuencasCatalanaToArcgis, mapperFromCuencasDueroToArcgis } from "./embalses.mappers.js"; +import { mapperFromCuencasMediterraneaToArcgis, mapperFromCuencasCantabricoToArcgis, mapperFromCuencasCatalanaToArcgis, mapperFromCuencasDueroToArcgis, mapperFromCuencasGuadalquivirToArcgis } from "./embalses.mappers.js"; import { scrapeCuencaMediterranea } from "scraping-cuenca-mediterranea"; import { scrapeCuencaCantabrica } from 'scraping-cuenca-cantabrico'; import { integracionCuencaCatalana } from 'scraping-cuenca-catalana'; import { getEstadoCuencaDuero } from 'scraping-cuenca-duero'; +import { scrapeCuencaGuadalquivir } from 'scraping-cuenca-guadalquivir'; import { parseDate } from "./embalses.helpers.js"; export const embalsesRepository = { @@ -231,6 +232,61 @@ export const embalsesRepository = { `Resumen Cuenca Duero: ${actualizados} actualizados, ${noEncontrados} no encontrados, ${sinMapper} sin mapper` ); + return actualizados > 0; + }, + actualizarCuencaGuadalquivir: async (): Promise => { + const embalsesGuadalquivir = await scrapeCuencaGuadalquivir(); + + console.log( + `Se han scrapeado ${embalsesGuadalquivir.length} embalses de la Cuenca Mediterránea` + ); + + let actualizados = 0; + let noEncontrados = 0; + let sinMapper = 0; + + for (const embalseGuadalquivirPorZonas of embalsesGuadalquivir) { + for (const embalse of embalseGuadalquivirPorZonas.embalses) { + const infoDestino = mapperFromCuencasGuadalquivirToArcgis.get(embalse.id); + + if (!infoDestino) { + sinMapper++; + console.warn(`Sin mapper para ID ${embalse.id} - ${embalse.nombre}`); + continue; + } + + console.log( + `🔍 Mapeando: ID scraping ${embalse.id} -> _id BD ${infoDestino.idArcgis} (${infoDestino.nombre})` + ); + + const { matchedCount } = await getEmbalsesContext().updateOne( + { _id: infoDestino.idArcgis.toString() }, + { + $set: { + aguaActualSAIH: embalse.aguaActualSAIH, + fechaMedidaAguaActualSAIH: parseDate(embalse.fechaMedidaSAIH), + }, + } + ); + + if (matchedCount > 0) { + actualizados++; + console.log( + `Actualizado: ${infoDestino.nombre} (_id: ${infoDestino.idArcgis}) -> ${embalse.aguaActualSAIH} hm³` + ); + } else { + noEncontrados++; + console.warn( + `No encontrado en BD: _id ${infoDestino.idArcgis} - ${infoDestino.nombre}` + ); + } + } + } + + console.log( + `Resumen Cuenca Duero: ${actualizados} actualizados, ${noEncontrados} no encontrados, ${sinMapper} sin mapper` + ); + return actualizados > 0; } };