From 2246b57c2a02b9ebda59bfb23f931555e92cab30 Mon Sep 17 00:00:00 2001 From: Nicola Belluti Date: Fri, 8 May 2026 14:10:34 +0200 Subject: [PATCH] Switched to per-day zipped CSVs with combined CSV included --- bun.lock | 29 +++++++++++++++++ package.json | 3 ++ src/lib/components/StationRow.svelte | 34 ++++++++++++-------- src/lib/csv-pool.js | 48 ---------------------------- src/lib/csv.js | 26 ++++++++------- src/lib/csv.worker.js | 5 --- 6 files changed, 66 insertions(+), 79 deletions(-) delete mode 100644 src/lib/csv-pool.js delete mode 100644 src/lib/csv.worker.js diff --git a/bun.lock b/bun.lock index 14f7164..f4906c9 100644 --- a/bun.lock +++ b/bun.lock @@ -4,6 +4,9 @@ "workspaces": { "": { "name": "download-dati-centraline", + "dependencies": { + "jszip": "^3.10.1", + }, "devDependencies": { "@sveltejs/adapter-auto": "^7.0.1", "@sveltejs/kit": "^2.57.0", @@ -100,6 +103,8 @@ "cookie": ["cookie@0.6.0", "", {}, "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw=="], + "core-util-is": ["core-util-is@1.0.3", "", {}, "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="], + "deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="], "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], @@ -114,10 +119,20 @@ "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "immediate": ["immediate@3.0.6", "", {}, "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ=="], + + "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + "is-reference": ["is-reference@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.6" } }, "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw=="], + "isarray": ["isarray@1.0.0", "", {}, "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="], + + "jszip": ["jszip@3.10.1", "", { "dependencies": { "lie": "~3.3.0", "pako": "~1.0.2", "readable-stream": "~2.3.6", "setimmediate": "^1.0.5" } }, "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g=="], + "kleur": ["kleur@4.1.5", "", {}, "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ=="], + "lie": ["lie@3.3.0", "", { "dependencies": { "immediate": "~3.0.5" } }, "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ=="], + "lightningcss": ["lightningcss@1.32.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.32.0", "lightningcss-darwin-arm64": "1.32.0", "lightningcss-darwin-x64": "1.32.0", "lightningcss-freebsd-x64": "1.32.0", "lightningcss-linux-arm-gnueabihf": "1.32.0", "lightningcss-linux-arm64-gnu": "1.32.0", "lightningcss-linux-arm64-musl": "1.32.0", "lightningcss-linux-x64-gnu": "1.32.0", "lightningcss-linux-x64-musl": "1.32.0", "lightningcss-win32-arm64-msvc": "1.32.0", "lightningcss-win32-x64-msvc": "1.32.0" } }, "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ=="], "lightningcss-android-arm64": ["lightningcss-android-arm64@1.32.0", "", { "os": "android", "cpu": "arm64" }, "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg=="], @@ -154,24 +169,36 @@ "obug": ["obug@2.1.1", "", {}, "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ=="], + "pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="], + "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], "picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="], "postcss": ["postcss@8.5.14", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg=="], + "process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="], + + "readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + "readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="], "rolldown": ["rolldown@1.0.0-rc.18", "", { "dependencies": { "@oxc-project/types": "=0.128.0", "@rolldown/pluginutils": "1.0.0-rc.18" }, "optionalDependencies": { "@rolldown/binding-android-arm64": "1.0.0-rc.18", "@rolldown/binding-darwin-arm64": "1.0.0-rc.18", "@rolldown/binding-darwin-x64": "1.0.0-rc.18", "@rolldown/binding-freebsd-x64": "1.0.0-rc.18", "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.18", "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.18", "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.18", "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.18", "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.18", "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.18", "@rolldown/binding-linux-x64-musl": "1.0.0-rc.18", "@rolldown/binding-openharmony-arm64": "1.0.0-rc.18", "@rolldown/binding-wasm32-wasi": "1.0.0-rc.18", "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.18", "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.18" }, "bin": { "rolldown": "bin/cli.mjs" } }, "sha512-phmyKBpuBdRYDf4hgyynGAYn/rDDe+iZXKVJ7WX5b1zQzpLkP5oJRPGsfJuHdzPMlyyEO/4sPW6yfSx2gf7lVg=="], "sade": ["sade@1.8.1", "", { "dependencies": { "mri": "^1.1.0" } }, "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A=="], + "safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], + "set-cookie-parser": ["set-cookie-parser@3.1.0", "", {}, "sha512-kjnC1DXBHcxaOaOXBHBeRtltsDG2nUiUni+jP92M9gYdW12rsmx92UsfpH7o5tDRs7I1ZZPSQJQGv3UaRfCiuw=="], + "setimmediate": ["setimmediate@1.0.5", "", {}, "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="], + "sirv": ["sirv@3.0.2", "", { "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", "totalist": "^3.0.0" } }, "sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g=="], "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + "string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + "svelte": ["svelte@5.55.5", "", { "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", "@sveltejs/acorn-typescript": "^1.0.5", "@types/estree": "^1.0.5", "@types/trusted-types": "^2.0.7", "acorn": "^8.12.1", "aria-query": "5.3.1", "axobject-query": "^4.1.0", "clsx": "^2.1.1", "devalue": "^5.6.4", "esm-env": "^1.2.1", "esrap": "^2.2.4", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", "zimmerframe": "^1.1.2" } }, "sha512-2uCs/LZ9us+AktdzYJM8OcxQ8qnPS1kpaO7syGT/MgO+6Qr1Ybl+TqPq+97u7PHqmmMlye5ZkoyXONy5mjjAbw=="], "svelte-check": ["svelte-check@4.4.8", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "chokidar": "^4.0.1", "fdir": "^6.2.0", "picocolors": "^1.0.0", "sade": "^1.7.4" }, "peerDependencies": { "svelte": "^4.0.0 || ^5.0.0-next.0", "typescript": ">=5.0.0" }, "bin": { "svelte-check": "bin/svelte-check" } }, "sha512-67adfgBox5eNSNIvIIwgFizKGdcRrGpiMoNO2obHcYuLz7iTa8Xgm/NGU3ntMFnNm8K1grFOIG6HhMLX/vcN8w=="], @@ -184,6 +211,8 @@ "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], + "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], + "vite": ["vite@8.0.11", "", { "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", "postcss": "^8.5.14", "rolldown": "1.0.0-rc.18", "tinyglobby": "^0.2.16" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.1.18", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-Jz1mxtUBR5xTT65VOdJZUUeoyLtqljmFkiUXhPTLZka3RDc9vpi/xXkyrnsdRcm2lIi3l3GPMnAidTsEGIj3Ow=="], "vitefu": ["vitefu@1.1.3", "", { "peerDependencies": { "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["vite"] }, "sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg=="], diff --git a/package.json b/package.json index b904a34..9b607d8 100644 --- a/package.json +++ b/package.json @@ -19,5 +19,8 @@ "svelte-check": "^4.4.6", "typescript": "^6.0.2", "vite": "^8.0.7" + }, + "dependencies": { + "jszip": "^3.10.1" } } diff --git a/src/lib/components/StationRow.svelte b/src/lib/components/StationRow.svelte index e17e13c..64cbbb3 100644 --- a/src/lib/components/StationRow.svelte +++ b/src/lib/components/StationRow.svelte @@ -2,7 +2,7 @@ import { onDestroy } from 'svelte'; import { getDayRange, formatDayParam } from '$lib/timezone.js'; import { fetchDay } from '$lib/api.js'; - import { downloadCsv, buildFilename } from '$lib/csv.js'; + import { buildCsv, downloadBlob, buildDayFilename, buildZipFilename } from '$lib/csv.js'; import { enqueue } from '$lib/download-pool.js'; let { station } = $props(); @@ -28,9 +28,7 @@ let pct = $derived(totalDays > 0 ? (currentDay / totalDays) * 100 : 0); async function startDownload() { - const { buildCsvAsync } = await import('$lib/csv-pool.js'); - // Un controller per sessione: abort() cancella tutte le fetch di questa centralina - // senza interferire con i download paralleli di altre stazioni. + const JSZip = (await import('jszip')).default; controller = new AbortController(); const { signal } = controller; downloading = true; @@ -39,8 +37,6 @@ totalDays = days.length; currentDay = 0; - // Array pre-allocato per indice: le fetch partono in parallelo e completano fuori ordine, - // ma resultsByDay[idx] garantisce che il CSV rispetti la sequenza temporale. const resultsByDay = new Array(days.length); await Promise.all(days.map((day, idx) => @@ -52,17 +48,27 @@ )); if (!signal.aborted) { - const measurements = resultsByDay.flat(); currentDay = totalDays; - // Breve pausa prima del reset: l'utente deve poter vedere la barra al 100%. await new Promise(r => setTimeout(r, 900)); - if (!signal.aborted && measurements.length > 0) { - const csv = await buildCsvAsync(measurements); - downloadCsv(csv, buildFilename(station.title, station.pk, new Date())); - completed = true; - // Il segno di spunta deve restare visibile abbastanza a lungo da essere notato. - await new Promise(r => setTimeout(r, 2000)); + if (!signal.aborted) { + const zip = new JSZip(); + let allMeasurements = []; + + for (let i = 0; i < days.length; i++) { + const measurements = resultsByDay[i]; + if (!measurements?.length) continue; + zip.file(buildDayFilename(days[i]), buildCsv(measurements, false)); + allMeasurements = allMeasurements.concat(measurements); + } + + if (allMeasurements.length > 0) { + zip.file(buildZipFilename(station.title, station.pk).replace('.zip', '.csv'), buildCsv(allMeasurements)); + const blob = await zip.generateAsync({ type: 'blob', compression: 'DEFLATE', compressionOptions: { level: 6 } }); + downloadBlob(blob, buildZipFilename(station.title, station.pk)); + completed = true; + await new Promise(r => setTimeout(r, 2000)); + } } } else { await new Promise(r => setTimeout(r, 700)); diff --git a/src/lib/csv-pool.js b/src/lib/csv-pool.js deleted file mode 100644 index 921aed0..0000000 --- a/src/lib/csv-pool.js +++ /dev/null @@ -1,48 +0,0 @@ -import CsvWorker from './csv.worker.js?worker'; - -// Costruire il CSV di un'intera centralina (potenzialmente decine di migliaia di righe) -// blocca il main thread per centinaia di ms: si delega a Worker separati. -// Due worker perché la costruzione CSV è CPU-bound: aggiungerne di più non migliora -// il singolo job ma aumenta il consumo di memoria. -const POOL_SIZE = 2; - -const queue = []; -const idle = []; - -for (let i = 0; i < POOL_SIZE; i++) { - const w = new CsvWorker(); - - w.onmessage = ({ data }) => { - // _task aggancia la Promise al Worker: il message handler non riceve l'identità - // del worker mittente, quindi è l'unico modo per risalire alla Promise da risolvere. - const { resolve } = w._task; - w._task = null; - idle.push(w); - resolve(data); - dispatch(); - }; - - w.onerror = e => { - const { reject } = w._task; - w._task = null; - idle.push(w); - reject(e); - dispatch(); - }; - - idle.push(w); -} - -function dispatch() { - if (!queue.length || !idle.length) return; - const w = idle.pop(); - w._task = queue.shift(); - w.postMessage(w._task.measurements); -} - -export function buildCsvAsync(measurements) { - return new Promise((resolve, reject) => { - queue.push({ measurements, resolve, reject }); - dispatch(); - }); -} diff --git a/src/lib/csv.js b/src/lib/csv.js index 447e06a..2ebdeb1 100644 --- a/src/lib/csv.js +++ b/src/lib/csv.js @@ -1,31 +1,33 @@ import { formatTimestampRome, ROME } from './timezone.js'; -export function buildCsv(measurements) { - const header = '"Data","Ora","Temperatura","Umidità","PM10"'; +export function buildCsv(measurements, includeDate = true) { + const header = includeDate + ? '"Data","Ora","Temperatura","Umidità","PM10"' + : '"Ora","Temperatura","Umidità","PM10"'; const rows = measurements.map(m => { const [date, time] = formatTimestampRome(m.time).split(' '); - return `"${date}","${time.split('.')[0]}",${m.temp},${m.umid},${m.pm10}`; + const t = time.split('.')[0]; + return includeDate ? `"${date}","${t}",${m.temp},${m.umid},${m.pm10}` : `"${t}",${m.temp},${m.umid},${m.pm10}`; }); // BOM obbligatorio: Excel su Windows interpreta UTF-8 senza BOM come Windows-1252, // corrompendo i caratteri accentati nelle intestazioni. return '' + [header, ...rows].join('\n'); } -export function downloadCsv(content, filename) { - const blob = new Blob([content], { type: 'text/csv;charset=utf-8;' }); +export function downloadBlob(blob, filename) { const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = filename; a.click(); - // revokeObjectURL libera subito la memoria del blob: il click sincrono è sufficiente - // perché il browser avvia il download prima di passare allo statement successivo. URL.revokeObjectURL(url); } -export function buildFilename(title, id, date) { - // en-CA produce YYYY-MM-DD, il formato ISO che vogliamo nel nome file. - const dateStr = new Intl.DateTimeFormat('en-CA', { timeZone: ROME }).format(date); - const sanitized = title.replace(/ /g, '_').replace(/[^a-zA-Z0-9_]/g, ''); - return `${sanitized}_(${id})_${dateStr}.csv`; +export function buildDayFilename(date) { + return new Intl.DateTimeFormat('en-CA', { timeZone: ROME }).format(date) + '.csv'; +} + +export function buildZipFilename(title, id) { + const sanitized = title.replace(/ /g, '_').replace(/[^a-zA-Z0-9_]/g, ''); + return `${sanitized}_(${id}).zip`; } diff --git a/src/lib/csv.worker.js b/src/lib/csv.worker.js deleted file mode 100644 index 74f5b1c..0000000 --- a/src/lib/csv.worker.js +++ /dev/null @@ -1,5 +0,0 @@ -import { buildCsv } from './csv.js'; - -self.onmessage = ({ data: measurements }) => { - self.postMessage(buildCsv(measurements)); -};