Switched to per-day zipped CSVs with combined CSV included

This commit is contained in:
2026-05-08 14:10:34 +02:00
parent 41152138c1
commit 2246b57c2a
6 changed files with 66 additions and 79 deletions
+29
View File
@@ -4,6 +4,9 @@
"workspaces": {
"": {
"name": "download-dati-centraline",
"dependencies": {
"jszip": "^3.10.1",
},
"devDependencies": {
"@sveltejs/adapter-auto": "^7.0.1",
"@sveltejs/kit": "^2.57.0",
@@ -100,6 +103,8 @@
"cookie": ["cookie@0.6.0", "", {}, "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw=="],
"core-util-is": ["core-util-is@1.0.3", "", {}, "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="],
"deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="],
"detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="],
@@ -114,10 +119,20 @@
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
"immediate": ["immediate@3.0.6", "", {}, "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ=="],
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
"is-reference": ["is-reference@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.6" } }, "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw=="],
"isarray": ["isarray@1.0.0", "", {}, "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="],
"jszip": ["jszip@3.10.1", "", { "dependencies": { "lie": "~3.3.0", "pako": "~1.0.2", "readable-stream": "~2.3.6", "setimmediate": "^1.0.5" } }, "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g=="],
"kleur": ["kleur@4.1.5", "", {}, "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ=="],
"lie": ["lie@3.3.0", "", { "dependencies": { "immediate": "~3.0.5" } }, "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ=="],
"lightningcss": ["lightningcss@1.32.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.32.0", "lightningcss-darwin-arm64": "1.32.0", "lightningcss-darwin-x64": "1.32.0", "lightningcss-freebsd-x64": "1.32.0", "lightningcss-linux-arm-gnueabihf": "1.32.0", "lightningcss-linux-arm64-gnu": "1.32.0", "lightningcss-linux-arm64-musl": "1.32.0", "lightningcss-linux-x64-gnu": "1.32.0", "lightningcss-linux-x64-musl": "1.32.0", "lightningcss-win32-arm64-msvc": "1.32.0", "lightningcss-win32-x64-msvc": "1.32.0" } }, "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ=="],
"lightningcss-android-arm64": ["lightningcss-android-arm64@1.32.0", "", { "os": "android", "cpu": "arm64" }, "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg=="],
@@ -154,24 +169,36 @@
"obug": ["obug@2.1.1", "", {}, "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ=="],
"pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="],
"picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],
"picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="],
"postcss": ["postcss@8.5.14", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg=="],
"process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="],
"readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
"readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="],
"rolldown": ["rolldown@1.0.0-rc.18", "", { "dependencies": { "@oxc-project/types": "=0.128.0", "@rolldown/pluginutils": "1.0.0-rc.18" }, "optionalDependencies": { "@rolldown/binding-android-arm64": "1.0.0-rc.18", "@rolldown/binding-darwin-arm64": "1.0.0-rc.18", "@rolldown/binding-darwin-x64": "1.0.0-rc.18", "@rolldown/binding-freebsd-x64": "1.0.0-rc.18", "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.18", "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.18", "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.18", "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.18", "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.18", "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.18", "@rolldown/binding-linux-x64-musl": "1.0.0-rc.18", "@rolldown/binding-openharmony-arm64": "1.0.0-rc.18", "@rolldown/binding-wasm32-wasi": "1.0.0-rc.18", "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.18", "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.18" }, "bin": { "rolldown": "bin/cli.mjs" } }, "sha512-phmyKBpuBdRYDf4hgyynGAYn/rDDe+iZXKVJ7WX5b1zQzpLkP5oJRPGsfJuHdzPMlyyEO/4sPW6yfSx2gf7lVg=="],
"sade": ["sade@1.8.1", "", { "dependencies": { "mri": "^1.1.0" } }, "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A=="],
"safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="],
"set-cookie-parser": ["set-cookie-parser@3.1.0", "", {}, "sha512-kjnC1DXBHcxaOaOXBHBeRtltsDG2nUiUni+jP92M9gYdW12rsmx92UsfpH7o5tDRs7I1ZZPSQJQGv3UaRfCiuw=="],
"setimmediate": ["setimmediate@1.0.5", "", {}, "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="],
"sirv": ["sirv@3.0.2", "", { "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", "totalist": "^3.0.0" } }, "sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g=="],
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
"string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
"svelte": ["svelte@5.55.5", "", { "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", "@sveltejs/acorn-typescript": "^1.0.5", "@types/estree": "^1.0.5", "@types/trusted-types": "^2.0.7", "acorn": "^8.12.1", "aria-query": "5.3.1", "axobject-query": "^4.1.0", "clsx": "^2.1.1", "devalue": "^5.6.4", "esm-env": "^1.2.1", "esrap": "^2.2.4", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", "zimmerframe": "^1.1.2" } }, "sha512-2uCs/LZ9us+AktdzYJM8OcxQ8qnPS1kpaO7syGT/MgO+6Qr1Ybl+TqPq+97u7PHqmmMlye5ZkoyXONy5mjjAbw=="],
"svelte-check": ["svelte-check@4.4.8", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "chokidar": "^4.0.1", "fdir": "^6.2.0", "picocolors": "^1.0.0", "sade": "^1.7.4" }, "peerDependencies": { "svelte": "^4.0.0 || ^5.0.0-next.0", "typescript": ">=5.0.0" }, "bin": { "svelte-check": "bin/svelte-check" } }, "sha512-67adfgBox5eNSNIvIIwgFizKGdcRrGpiMoNO2obHcYuLz7iTa8Xgm/NGU3ntMFnNm8K1grFOIG6HhMLX/vcN8w=="],
@@ -184,6 +211,8 @@
"typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="],
"util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="],
"vite": ["vite@8.0.11", "", { "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", "postcss": "^8.5.14", "rolldown": "1.0.0-rc.18", "tinyglobby": "^0.2.16" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.1.18", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-Jz1mxtUBR5xTT65VOdJZUUeoyLtqljmFkiUXhPTLZka3RDc9vpi/xXkyrnsdRcm2lIi3l3GPMnAidTsEGIj3Ow=="],
"vitefu": ["vitefu@1.1.3", "", { "peerDependencies": { "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["vite"] }, "sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg=="],
+3
View File
@@ -19,5 +19,8 @@
"svelte-check": "^4.4.6",
"typescript": "^6.0.2",
"vite": "^8.0.7"
},
"dependencies": {
"jszip": "^3.10.1"
}
}
+18 -12
View File
@@ -2,7 +2,7 @@
import { onDestroy } from 'svelte';
import { getDayRange, formatDayParam } from '$lib/timezone.js';
import { fetchDay } from '$lib/api.js';
import { downloadCsv, buildFilename } from '$lib/csv.js';
import { buildCsv, downloadBlob, buildDayFilename, buildZipFilename } from '$lib/csv.js';
import { enqueue } from '$lib/download-pool.js';
let { station } = $props();
@@ -28,9 +28,7 @@
let pct = $derived(totalDays > 0 ? (currentDay / totalDays) * 100 : 0);
async function startDownload() {
const { buildCsvAsync } = await import('$lib/csv-pool.js');
// Un controller per sessione: abort() cancella tutte le fetch di questa centralina
// senza interferire con i download paralleli di altre stazioni.
const JSZip = (await import('jszip')).default;
controller = new AbortController();
const { signal } = controller;
downloading = true;
@@ -39,8 +37,6 @@
totalDays = days.length;
currentDay = 0;
// Array pre-allocato per indice: le fetch partono in parallelo e completano fuori ordine,
// ma resultsByDay[idx] garantisce che il CSV rispetti la sequenza temporale.
const resultsByDay = new Array(days.length);
await Promise.all(days.map((day, idx) =>
@@ -52,18 +48,28 @@
));
if (!signal.aborted) {
const measurements = resultsByDay.flat();
currentDay = totalDays;
// Breve pausa prima del reset: l'utente deve poter vedere la barra al 100%.
await new Promise(r => setTimeout(r, 900));
if (!signal.aborted && measurements.length > 0) {
const csv = await buildCsvAsync(measurements);
downloadCsv(csv, buildFilename(station.title, station.pk, new Date()));
if (!signal.aborted) {
const zip = new JSZip();
let allMeasurements = [];
for (let i = 0; i < days.length; i++) {
const measurements = resultsByDay[i];
if (!measurements?.length) continue;
zip.file(buildDayFilename(days[i]), buildCsv(measurements, false));
allMeasurements = allMeasurements.concat(measurements);
}
if (allMeasurements.length > 0) {
zip.file(buildZipFilename(station.title, station.pk).replace('.zip', '.csv'), buildCsv(allMeasurements));
const blob = await zip.generateAsync({ type: 'blob', compression: 'DEFLATE', compressionOptions: { level: 6 } });
downloadBlob(blob, buildZipFilename(station.title, station.pk));
completed = true;
// Il segno di spunta deve restare visibile abbastanza a lungo da essere notato.
await new Promise(r => setTimeout(r, 2000));
}
}
} else {
await new Promise(r => setTimeout(r, 700));
}
-48
View File
@@ -1,48 +0,0 @@
import CsvWorker from './csv.worker.js?worker';
// Costruire il CSV di un'intera centralina (potenzialmente decine di migliaia di righe)
// blocca il main thread per centinaia di ms: si delega a Worker separati.
// Due worker perché la costruzione CSV è CPU-bound: aggiungerne di più non migliora
// il singolo job ma aumenta il consumo di memoria.
const POOL_SIZE = 2;
const queue = [];
const idle = [];
for (let i = 0; i < POOL_SIZE; i++) {
const w = new CsvWorker();
w.onmessage = ({ data }) => {
// _task aggancia la Promise al Worker: il message handler non riceve l'identità
// del worker mittente, quindi è l'unico modo per risalire alla Promise da risolvere.
const { resolve } = w._task;
w._task = null;
idle.push(w);
resolve(data);
dispatch();
};
w.onerror = e => {
const { reject } = w._task;
w._task = null;
idle.push(w);
reject(e);
dispatch();
};
idle.push(w);
}
function dispatch() {
if (!queue.length || !idle.length) return;
const w = idle.pop();
w._task = queue.shift();
w.postMessage(w._task.measurements);
}
export function buildCsvAsync(measurements) {
return new Promise((resolve, reject) => {
queue.push({ measurements, resolve, reject });
dispatch();
});
}
+14 -12
View File
@@ -1,31 +1,33 @@
import { formatTimestampRome, ROME } from './timezone.js';
export function buildCsv(measurements) {
const header = '"Data","Ora","Temperatura","Umidità","PM10"';
export function buildCsv(measurements, includeDate = true) {
const header = includeDate
? '"Data","Ora","Temperatura","Umidità","PM10"'
: '"Ora","Temperatura","Umidità","PM10"';
const rows = measurements.map(m => {
const [date, time] = formatTimestampRome(m.time).split(' ');
return `"${date}","${time.split('.')[0]}",${m.temp},${m.umid},${m.pm10}`;
const t = time.split('.')[0];
return includeDate ? `"${date}","${t}",${m.temp},${m.umid},${m.pm10}` : `"${t}",${m.temp},${m.umid},${m.pm10}`;
});
// BOM obbligatorio: Excel su Windows interpreta UTF-8 senza BOM come Windows-1252,
// corrompendo i caratteri accentati nelle intestazioni.
return '' + [header, ...rows].join('\n');
}
export function downloadCsv(content, filename) {
const blob = new Blob([content], { type: 'text/csv;charset=utf-8;' });
export function downloadBlob(blob, filename) {
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = filename;
a.click();
// revokeObjectURL libera subito la memoria del blob: il click sincrono è sufficiente
// perché il browser avvia il download prima di passare allo statement successivo.
URL.revokeObjectURL(url);
}
export function buildFilename(title, id, date) {
// en-CA produce YYYY-MM-DD, il formato ISO che vogliamo nel nome file.
const dateStr = new Intl.DateTimeFormat('en-CA', { timeZone: ROME }).format(date);
const sanitized = title.replace(/ /g, '_').replace(/[^a-zA-Z0-9_]/g, '');
return `${sanitized}_(${id})_${dateStr}.csv`;
export function buildDayFilename(date) {
return new Intl.DateTimeFormat('en-CA', { timeZone: ROME }).format(date) + '.csv';
}
export function buildZipFilename(title, id) {
const sanitized = title.replace(/ /g, '_').replace(/[^a-zA-Z0-9_]/g, '');
return `${sanitized}_(${id}).zip`;
}
-5
View File
@@ -1,5 +0,0 @@
import { buildCsv } from './csv.js';
self.onmessage = ({ data: measurements }) => {
self.postMessage(buildCsv(measurements));
};