feat(map): persist city building tiles
This commit is contained in:
@@ -0,0 +1,179 @@
|
|||||||
|
import { EtagMismatch, type RangeResponse, type Source } from "pmtiles";
|
||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import {
|
||||||
|
PersistentCityPmtilesSource,
|
||||||
|
type CityPmtilesRangeRecord,
|
||||||
|
type CityPmtilesRangeStore
|
||||||
|
} from "./city-pmtiles-cache";
|
||||||
|
|
||||||
|
class MemoryRangeStore implements CityPmtilesRangeStore {
|
||||||
|
readonly records = new Map<string, CityPmtilesRangeRecord>();
|
||||||
|
readonly revisions = new Map<string, string>();
|
||||||
|
|
||||||
|
async get(key: string) {
|
||||||
|
return this.records.get(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
async put(record: CityPmtilesRangeRecord) {
|
||||||
|
if (this.revisions.get(record.url) !== record.revision) return;
|
||||||
|
this.records.set(record.key, record);
|
||||||
|
}
|
||||||
|
|
||||||
|
async prepareRevision(url: string, revision: string) {
|
||||||
|
if (this.revisions.get(url) === revision) return;
|
||||||
|
for (const [key, record] of this.records) {
|
||||||
|
if (record.url === url && record.revision !== revision) this.records.delete(key);
|
||||||
|
}
|
||||||
|
this.revisions.set(url, revision);
|
||||||
|
}
|
||||||
|
|
||||||
|
async deleteRevision(url: string, revision: string) {
|
||||||
|
for (const [key, record] of this.records) {
|
||||||
|
if (record.url === url && record.revision === revision) this.records.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function upstream(data: number[], etag = '"scene-v1"') {
|
||||||
|
const getBytes = vi.fn<Source["getBytes"]>(async () => ({
|
||||||
|
data: new Uint8Array(data).buffer,
|
||||||
|
etag,
|
||||||
|
cacheControl: "public, max-age=3600"
|
||||||
|
}));
|
||||||
|
return { getBytes, getKey: () => "upstream" } satisfies Source;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("PersistentCityPmtilesSource", () => {
|
||||||
|
it("reuses persisted byte ranges across source instances", async () => {
|
||||||
|
const store = new MemoryRangeStore();
|
||||||
|
const firstUpstream = upstream([1, 2, 3, 4]);
|
||||||
|
const first = new PersistentCityPmtilesSource(
|
||||||
|
"https://example.test/buildings.pmtiles",
|
||||||
|
"scene-v1",
|
||||||
|
firstUpstream,
|
||||||
|
store
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(first.getBytes(100, 4)).resolves.toMatchObject({ etag: '"scene-v1"' });
|
||||||
|
|
||||||
|
const secondUpstream = upstream([9, 9, 9, 9]);
|
||||||
|
const second = new PersistentCityPmtilesSource(
|
||||||
|
"https://example.test/buildings.pmtiles",
|
||||||
|
"scene-v1",
|
||||||
|
secondUpstream,
|
||||||
|
store
|
||||||
|
);
|
||||||
|
const cached = await second.getBytes(100, 4);
|
||||||
|
|
||||||
|
expect([...new Uint8Array(cached.data)]).toEqual([1, 2, 3, 4]);
|
||||||
|
expect(firstUpstream.getBytes).toHaveBeenCalledOnce();
|
||||||
|
expect(secondUpstream.getBytes).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("removes old revision ranges before reading a rebuilt archive", async () => {
|
||||||
|
const store = new MemoryRangeStore();
|
||||||
|
const oldSource = new PersistentCityPmtilesSource(
|
||||||
|
"https://example.test/buildings.pmtiles",
|
||||||
|
"scene-v1",
|
||||||
|
upstream([1, 2]),
|
||||||
|
store
|
||||||
|
);
|
||||||
|
await oldSource.getBytes(0, 2);
|
||||||
|
|
||||||
|
const newUpstream = upstream([3, 4], '"scene-v2"');
|
||||||
|
const newSource = new PersistentCityPmtilesSource(
|
||||||
|
"https://example.test/buildings.pmtiles",
|
||||||
|
"scene-v2",
|
||||||
|
newUpstream,
|
||||||
|
store
|
||||||
|
);
|
||||||
|
const response = await newSource.getBytes(0, 2);
|
||||||
|
|
||||||
|
expect([...new Uint8Array(response.data)]).toEqual([3, 4]);
|
||||||
|
expect(newUpstream.getBytes).toHaveBeenCalledOnce();
|
||||||
|
expect([...store.records.values()].map((record) => record.revision)).toEqual(["scene-v2"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to the network when persistent storage is unavailable", async () => {
|
||||||
|
const unavailableStore: CityPmtilesRangeStore = {
|
||||||
|
get: vi.fn(async () => {
|
||||||
|
throw new DOMException("Quota exceeded", "QuotaExceededError");
|
||||||
|
}),
|
||||||
|
put: vi.fn(async () => {
|
||||||
|
throw new DOMException("Quota exceeded", "QuotaExceededError");
|
||||||
|
}),
|
||||||
|
prepareRevision: vi.fn(async () => {
|
||||||
|
throw new DOMException("IndexedDB unavailable", "InvalidStateError");
|
||||||
|
}),
|
||||||
|
deleteRevision: vi.fn(async () => {
|
||||||
|
throw new DOMException("IndexedDB unavailable", "InvalidStateError");
|
||||||
|
})
|
||||||
|
};
|
||||||
|
const network = upstream([7, 8, 9]);
|
||||||
|
const source = new PersistentCityPmtilesSource(
|
||||||
|
"https://example.test/buildings.pmtiles",
|
||||||
|
"scene-v1",
|
||||||
|
network,
|
||||||
|
unavailableStore
|
||||||
|
);
|
||||||
|
|
||||||
|
const response = await source.getBytes(0, 3);
|
||||||
|
|
||||||
|
expect([...new Uint8Array(response.data)]).toEqual([7, 8, 9]);
|
||||||
|
expect(network.getBytes).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not persist a truncated Range response", async () => {
|
||||||
|
const store = new MemoryRangeStore();
|
||||||
|
const network: Source = {
|
||||||
|
getKey: () => "upstream",
|
||||||
|
getBytes: vi.fn(async (): Promise<RangeResponse> => ({
|
||||||
|
data: new Uint8Array([1, 2]).buffer
|
||||||
|
}))
|
||||||
|
};
|
||||||
|
const source = new PersistentCityPmtilesSource(
|
||||||
|
"https://example.test/buildings.pmtiles",
|
||||||
|
"scene-v1",
|
||||||
|
network,
|
||||||
|
store
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(source.getBytes(0, 4)).rejects.toThrow("returned 2 bytes instead of 4");
|
||||||
|
expect(store.records.size).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accepts a complete small archive returned for the initial header range", async () => {
|
||||||
|
const bytes = new Array(256).fill(3);
|
||||||
|
const source = new PersistentCityPmtilesSource(
|
||||||
|
"https://example.test/small.pmtiles",
|
||||||
|
"small-v1",
|
||||||
|
upstream(bytes),
|
||||||
|
new MemoryRangeStore()
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(source.getBytes(0, 16_384)).resolves.toMatchObject({
|
||||||
|
data: expect.objectContaining({ byteLength: 256 })
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("purges the current revision before PMTiles retries an ETag mismatch", async () => {
|
||||||
|
const store = new MemoryRangeStore();
|
||||||
|
const url = "https://example.test/buildings.pmtiles";
|
||||||
|
const first = new PersistentCityPmtilesSource(url, "scene-v2", upstream([1, 2]), store);
|
||||||
|
await first.getBytes(0, 2);
|
||||||
|
expect(store.records.size).toBe(1);
|
||||||
|
|
||||||
|
const mismatchedUpstream: Source = {
|
||||||
|
getKey: () => url,
|
||||||
|
getBytes: vi.fn(async () => {
|
||||||
|
throw new EtagMismatch("rolling deployment changed the archive");
|
||||||
|
})
|
||||||
|
};
|
||||||
|
const second = new PersistentCityPmtilesSource(url, "scene-v2", mismatchedUpstream, store);
|
||||||
|
|
||||||
|
await expect(second.getBytes(2, 2, undefined, '"old-etag"')).rejects.toBeInstanceOf(
|
||||||
|
EtagMismatch
|
||||||
|
);
|
||||||
|
expect(store.records.size).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,347 @@
|
|||||||
|
import {
|
||||||
|
EtagMismatch,
|
||||||
|
FetchSource,
|
||||||
|
PMTiles,
|
||||||
|
type Protocol,
|
||||||
|
type RangeResponse,
|
||||||
|
type Source
|
||||||
|
} from "pmtiles";
|
||||||
|
|
||||||
|
const CACHE_DATABASE_NAME = "tjwater-city-pmtiles-cache";
|
||||||
|
const CACHE_DATABASE_VERSION = 2;
|
||||||
|
const CACHE_STORE_NAME = "ranges";
|
||||||
|
const CACHE_URL_INDEX = "url";
|
||||||
|
const CACHE_ARCHIVE_STORE_NAME = "archives";
|
||||||
|
const MAX_ARCHIVE_CACHE_BYTES = 48 * 1024 * 1024;
|
||||||
|
const PMTILES_HEADER_SIZE = 127;
|
||||||
|
|
||||||
|
export interface CityPmtilesRangeRecord extends RangeResponse {
|
||||||
|
key: string;
|
||||||
|
url: string;
|
||||||
|
revision: string;
|
||||||
|
offset: number;
|
||||||
|
length: number;
|
||||||
|
storedAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CityPmtilesRangeStore {
|
||||||
|
get(key: string): Promise<CityPmtilesRangeRecord | undefined>;
|
||||||
|
put(record: CityPmtilesRangeRecord): Promise<void>;
|
||||||
|
prepareRevision(url: string, revision: string): Promise<void>;
|
||||||
|
deleteRevision(url: string, revision: string): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CityPmtilesArchiveRecord {
|
||||||
|
url: string;
|
||||||
|
revision: string;
|
||||||
|
cachedBytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function abortIfRequested(signal?: AbortSignal) {
|
||||||
|
if (signal?.aborted) {
|
||||||
|
throw signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function requestResult<T>(request: IDBRequest<T>): Promise<T> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
request.addEventListener("success", () => resolve(request.result), { once: true });
|
||||||
|
request.addEventListener("error", () => reject(request.error), { once: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function transactionDone(transaction: IDBTransaction): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
transaction.addEventListener("complete", () => resolve(), { once: true });
|
||||||
|
transaction.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => reject(transaction.error ?? new Error("City PMTiles cache transaction aborted")),
|
||||||
|
{ once: true }
|
||||||
|
);
|
||||||
|
transaction.addEventListener(
|
||||||
|
"error",
|
||||||
|
() => reject(transaction.error ?? new Error("City PMTiles cache transaction failed")),
|
||||||
|
{ once: true }
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
class IndexedDbCityPmtilesRangeStore implements CityPmtilesRangeStore {
|
||||||
|
private readonly databasePromise: Promise<IDBDatabase>;
|
||||||
|
|
||||||
|
constructor(indexedDb: IDBFactory) {
|
||||||
|
this.databasePromise = new Promise((resolve, reject) => {
|
||||||
|
const request = indexedDb.open(CACHE_DATABASE_NAME, CACHE_DATABASE_VERSION);
|
||||||
|
request.addEventListener(
|
||||||
|
"upgradeneeded",
|
||||||
|
() => {
|
||||||
|
const database = request.result;
|
||||||
|
const transaction = request.transaction!;
|
||||||
|
const store = database.objectStoreNames.contains(CACHE_STORE_NAME)
|
||||||
|
? transaction.objectStore(CACHE_STORE_NAME)
|
||||||
|
: database.createObjectStore(CACHE_STORE_NAME, { keyPath: "key" });
|
||||||
|
if (!store.indexNames.contains(CACHE_URL_INDEX)) {
|
||||||
|
store.createIndex(CACHE_URL_INDEX, "url", { unique: false });
|
||||||
|
}
|
||||||
|
if (!database.objectStoreNames.contains(CACHE_ARCHIVE_STORE_NAME)) {
|
||||||
|
database.createObjectStore(CACHE_ARCHIVE_STORE_NAME, { keyPath: "url" });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{ once: true }
|
||||||
|
);
|
||||||
|
request.addEventListener(
|
||||||
|
"success",
|
||||||
|
() => {
|
||||||
|
const database = request.result;
|
||||||
|
database.addEventListener("versionchange", () => database.close());
|
||||||
|
resolve(database);
|
||||||
|
},
|
||||||
|
{ once: true }
|
||||||
|
);
|
||||||
|
request.addEventListener("error", () => reject(request.error), { once: true });
|
||||||
|
request.addEventListener(
|
||||||
|
"blocked",
|
||||||
|
() => reject(new Error("City PMTiles cache database upgrade was blocked")),
|
||||||
|
{ once: true }
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async get(key: string) {
|
||||||
|
const database = await this.databasePromise;
|
||||||
|
const transaction = database.transaction(CACHE_STORE_NAME, "readonly");
|
||||||
|
const result = requestResult(
|
||||||
|
transaction.objectStore(CACHE_STORE_NAME).get(key) as IDBRequest<
|
||||||
|
CityPmtilesRangeRecord | undefined
|
||||||
|
>
|
||||||
|
);
|
||||||
|
const [record] = await Promise.all([result, transactionDone(transaction)]);
|
||||||
|
return record;
|
||||||
|
}
|
||||||
|
|
||||||
|
async put(record: CityPmtilesRangeRecord) {
|
||||||
|
const database = await this.databasePromise;
|
||||||
|
const transaction = database.transaction(
|
||||||
|
[CACHE_STORE_NAME, CACHE_ARCHIVE_STORE_NAME],
|
||||||
|
"readwrite"
|
||||||
|
);
|
||||||
|
const ranges = transaction.objectStore(CACHE_STORE_NAME);
|
||||||
|
const archives = transaction.objectStore(CACHE_ARCHIVE_STORE_NAME);
|
||||||
|
const archiveRequest = archives.get(record.url) as IDBRequest<
|
||||||
|
CityPmtilesArchiveRecord | undefined
|
||||||
|
>;
|
||||||
|
archiveRequest.addEventListener("success", () => {
|
||||||
|
const archive = archiveRequest.result;
|
||||||
|
if (!archive || archive.revision !== record.revision) return;
|
||||||
|
const existingRequest = ranges.get(record.key) as IDBRequest<
|
||||||
|
CityPmtilesRangeRecord | undefined
|
||||||
|
>;
|
||||||
|
existingRequest.addEventListener("success", () => {
|
||||||
|
const cachedBytes =
|
||||||
|
archive.cachedBytes -
|
||||||
|
(existingRequest.result?.data.byteLength ?? 0) +
|
||||||
|
record.data.byteLength;
|
||||||
|
if (cachedBytes > MAX_ARCHIVE_CACHE_BYTES) return;
|
||||||
|
ranges.put(record);
|
||||||
|
archives.put({ ...archive, cachedBytes });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
await transactionDone(transaction);
|
||||||
|
}
|
||||||
|
|
||||||
|
async prepareRevision(url: string, revision: string) {
|
||||||
|
const database = await this.databasePromise;
|
||||||
|
const transaction = database.transaction(
|
||||||
|
[CACHE_STORE_NAME, CACHE_ARCHIVE_STORE_NAME],
|
||||||
|
"readwrite"
|
||||||
|
);
|
||||||
|
const ranges = transaction.objectStore(CACHE_STORE_NAME);
|
||||||
|
const archives = transaction.objectStore(CACHE_ARCHIVE_STORE_NAME);
|
||||||
|
const archiveRequest = archives.get(url) as IDBRequest<CityPmtilesArchiveRecord | undefined>;
|
||||||
|
archiveRequest.addEventListener("success", () => {
|
||||||
|
if (archiveRequest.result?.revision === revision) return;
|
||||||
|
let cachedBytes = 0;
|
||||||
|
const cursorRequest = ranges.index(CACHE_URL_INDEX).openCursor(IDBKeyRange.only(url));
|
||||||
|
cursorRequest.addEventListener("success", () => {
|
||||||
|
const cursor = cursorRequest.result;
|
||||||
|
if (!cursor) {
|
||||||
|
archives.put({ url, revision, cachedBytes });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const record = cursor.value as CityPmtilesRangeRecord;
|
||||||
|
if (record.revision === revision) cachedBytes += record.data.byteLength;
|
||||||
|
else cursor.delete();
|
||||||
|
cursor.continue();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
await transactionDone(transaction);
|
||||||
|
}
|
||||||
|
|
||||||
|
async deleteRevision(url: string, revision: string) {
|
||||||
|
const database = await this.databasePromise;
|
||||||
|
const transaction = database.transaction(
|
||||||
|
[CACHE_STORE_NAME, CACHE_ARCHIVE_STORE_NAME],
|
||||||
|
"readwrite"
|
||||||
|
);
|
||||||
|
const ranges = transaction.objectStore(CACHE_STORE_NAME);
|
||||||
|
const archives = transaction.objectStore(CACHE_ARCHIVE_STORE_NAME);
|
||||||
|
const index = ranges.index(CACHE_URL_INDEX);
|
||||||
|
const request = index.openCursor(IDBKeyRange.only(url));
|
||||||
|
request.addEventListener("success", () => {
|
||||||
|
const cursor = request.result;
|
||||||
|
if (!cursor) {
|
||||||
|
const archiveRequest = archives.get(url) as IDBRequest<
|
||||||
|
CityPmtilesArchiveRecord | undefined
|
||||||
|
>;
|
||||||
|
archiveRequest.addEventListener("success", () => {
|
||||||
|
if (archiveRequest.result?.revision === revision) {
|
||||||
|
archives.put({ url, revision, cachedBytes: 0 });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const record = cursor.value as CityPmtilesRangeRecord;
|
||||||
|
if (record.revision === revision) cursor.delete();
|
||||||
|
cursor.continue();
|
||||||
|
});
|
||||||
|
await transactionDone(transaction);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let sharedPersistentStore: CityPmtilesRangeStore | null | undefined;
|
||||||
|
|
||||||
|
function createPersistentStore(): CityPmtilesRangeStore | undefined {
|
||||||
|
if (sharedPersistentStore === undefined) {
|
||||||
|
sharedPersistentStore =
|
||||||
|
"indexedDB" in globalThis && globalThis.indexedDB
|
||||||
|
? new IndexedDbCityPmtilesRangeStore(globalThis.indexedDB)
|
||||||
|
: null;
|
||||||
|
}
|
||||||
|
return sharedPersistentStore ?? undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rangeKey(url: string, revision: string, offset: number, length: number) {
|
||||||
|
return JSON.stringify([url, revision, offset, length]);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isValidRangeLength(offset: number, requestedLength: number, actualLength: number) {
|
||||||
|
return (
|
||||||
|
actualLength === requestedLength ||
|
||||||
|
(offset === 0 && actualLength >= PMTILES_HEADER_SIZE && actualLength < requestedLength)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Persists the exact byte ranges PMTiles requests. The manifest revision keeps
|
||||||
|
* an updated archive from reusing stale header, directory, or tile bytes.
|
||||||
|
*/
|
||||||
|
export class PersistentCityPmtilesSource implements Source {
|
||||||
|
private store: CityPmtilesRangeStore | undefined;
|
||||||
|
private readonly prepareStore: Promise<void> | undefined;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly url: string,
|
||||||
|
private readonly revision: string,
|
||||||
|
private readonly upstream: Source = new FetchSource(url),
|
||||||
|
store: CityPmtilesRangeStore | null | undefined = createPersistentStore()
|
||||||
|
) {
|
||||||
|
this.store = store ?? undefined;
|
||||||
|
this.prepareStore = this.store
|
||||||
|
?.prepareRevision(this.url, this.revision)
|
||||||
|
.catch(() => this.disableStore());
|
||||||
|
}
|
||||||
|
|
||||||
|
getKey() {
|
||||||
|
return this.url;
|
||||||
|
}
|
||||||
|
|
||||||
|
async getBytes(offset: number, length: number, signal?: AbortSignal, etag?: string) {
|
||||||
|
abortIfRequested(signal);
|
||||||
|
await this.prepareStore;
|
||||||
|
abortIfRequested(signal);
|
||||||
|
|
||||||
|
const key = rangeKey(this.url, this.revision, offset, length);
|
||||||
|
const cached = await this.readCachedRange(key);
|
||||||
|
abortIfRequested(signal);
|
||||||
|
if (
|
||||||
|
cached &&
|
||||||
|
isValidRangeLength(offset, length, cached.data.byteLength) &&
|
||||||
|
(!etag || cached.etag === etag)
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
data: cached.data.slice(0),
|
||||||
|
etag: cached.etag,
|
||||||
|
cacheControl: cached.cacheControl,
|
||||||
|
expires: cached.expires
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let response: RangeResponse;
|
||||||
|
try {
|
||||||
|
response = await this.upstream.getBytes(offset, length, signal, etag);
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof EtagMismatch) await this.clearCurrentRevision();
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
abortIfRequested(signal);
|
||||||
|
if (!isValidRangeLength(offset, length, response.data.byteLength)) {
|
||||||
|
throw new Error(
|
||||||
|
`City PMTiles range ${offset}-${offset + length - 1} returned ${response.data.byteLength} bytes instead of ${length}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
void this.writeCachedRange({
|
||||||
|
...response,
|
||||||
|
data: response.data.slice(0),
|
||||||
|
etag: response.etag ?? etag,
|
||||||
|
key,
|
||||||
|
url: this.url,
|
||||||
|
revision: this.revision,
|
||||||
|
offset,
|
||||||
|
length,
|
||||||
|
storedAt: Date.now()
|
||||||
|
});
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async readCachedRange(key: string) {
|
||||||
|
if (!this.store) return undefined;
|
||||||
|
try {
|
||||||
|
return await this.store.get(key);
|
||||||
|
} catch {
|
||||||
|
this.disableStore();
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async writeCachedRange(record: CityPmtilesRangeRecord) {
|
||||||
|
if (!this.store) return;
|
||||||
|
try {
|
||||||
|
await this.store.put(record);
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof DOMException && error.name === "QuotaExceededError") {
|
||||||
|
await this.clearCurrentRevision();
|
||||||
|
}
|
||||||
|
this.disableStore();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async clearCurrentRevision() {
|
||||||
|
if (!this.store) return;
|
||||||
|
try {
|
||||||
|
await this.store.deleteRevision(this.url, this.revision);
|
||||||
|
} catch {
|
||||||
|
this.disableStore();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private disableStore() {
|
||||||
|
this.store = undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function registerCityPmtiles(protocol: Protocol, url: string, revision?: string) {
|
||||||
|
protocol.add(
|
||||||
|
new PMTiles(revision ? new PersistentCityPmtilesSource(url, revision) : new FetchSource(url))
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -4,11 +4,13 @@ import {
|
|||||||
CITY_LAYERS,
|
CITY_LAYERS,
|
||||||
CITY_LIGHT,
|
CITY_LIGHT,
|
||||||
CITY_ROOT,
|
CITY_ROOT,
|
||||||
|
cityBuildingArchiveUrl,
|
||||||
cityManifestSchema,
|
cityManifestSchema,
|
||||||
citySources,
|
citySources,
|
||||||
findCitySceneInsertionLayerId
|
findCitySceneInsertionLayerId
|
||||||
} from "./city-scene-layers";
|
} from "./city-scene-layers";
|
||||||
import { Protocol } from "pmtiles";
|
import { Protocol } from "pmtiles";
|
||||||
|
import { registerCityPmtiles } from "./city-pmtiles-cache";
|
||||||
|
|
||||||
const cityTileProtocol = new Protocol();
|
const cityTileProtocol = new Protocol();
|
||||||
maplibregl.addProtocol("pmtiles", cityTileProtocol.tile);
|
maplibregl.addProtocol("pmtiles", cityTileProtocol.tile);
|
||||||
@@ -72,11 +74,13 @@ class CitySceneControl {
|
|||||||
export function installCityScene(map: MapLibreMap): () => void {
|
export function installCityScene(map: MapLibreMap): () => void {
|
||||||
const abort = new AbortController();
|
const abort = new AbortController();
|
||||||
let control: CitySceneControl | undefined;
|
let control: CitySceneControl | undefined;
|
||||||
void fetch(`${CITY_ROOT}/manifest.json`, { signal: abort.signal })
|
void fetch(`${CITY_ROOT}/manifest.json`, { cache: "no-cache", signal: abort.signal })
|
||||||
.then(async (response) => {
|
.then(async (response) => {
|
||||||
if (!response.ok) return;
|
if (!response.ok) return;
|
||||||
const manifest = cityManifestSchema.safeParse(await response.json());
|
const manifest = cityManifestSchema.safeParse(await response.json());
|
||||||
if (!manifest.success || abort.signal.aborted) return;
|
if (!manifest.success || abort.signal.aborted) return;
|
||||||
|
const archiveRevision = manifest.data.tiles?.buildings.sha256;
|
||||||
|
registerCityPmtiles(cityTileProtocol, cityBuildingArchiveUrl(), archiveRevision);
|
||||||
control = new CitySceneControl();
|
control = new CitySceneControl();
|
||||||
control.attach(map);
|
control.attach(map);
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";
|
|||||||
import {
|
import {
|
||||||
CITY_LAYERS,
|
CITY_LAYERS,
|
||||||
CITY_LIGHT,
|
CITY_LIGHT,
|
||||||
|
cityBuildingArchiveUrl,
|
||||||
cityManifestSchema,
|
cityManifestSchema,
|
||||||
citySources,
|
citySources,
|
||||||
findCitySceneInsertionLayerId
|
findCitySceneInsertionLayerId
|
||||||
@@ -11,12 +12,18 @@ import {
|
|||||||
const manifest = {
|
const manifest = {
|
||||||
schema_version: 1,
|
schema_version: 1,
|
||||||
name: "奉贤",
|
name: "奉贤",
|
||||||
|
generated_at: "2026-09-22T09:09:36.585868+00:00",
|
||||||
bbox: [121.34, 30.68, 121.85, 31.02],
|
bbox: [121.34, 30.68, 121.85, 31.02],
|
||||||
building_count: 1,
|
building_count: 1,
|
||||||
pipe_count: 1,
|
pipe_count: 1,
|
||||||
asset_count: 1,
|
asset_count: 1,
|
||||||
notice: "参考模型",
|
notice: "参考模型",
|
||||||
attribution: "OSM"
|
attribution: "OSM",
|
||||||
|
tiles: {
|
||||||
|
buildings: {
|
||||||
|
sha256: "52d5ce787f7b42eaf0d7f3a3acaa5789f535ff31921c0e109a8c71c1963428d3"
|
||||||
|
}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
describe("city scene geographic contract", () => {
|
describe("city scene geographic contract", () => {
|
||||||
it("inserts buildings after business layers and before transparent hit layers", () => {
|
it("inserts buildings after business layers and before transparent hit layers", () => {
|
||||||
@@ -41,6 +48,11 @@ describe("city scene geographic contract", () => {
|
|||||||
expect(Object.keys(citySources())).toEqual(["fengxian-city-buildings"]);
|
expect(Object.keys(citySources())).toEqual(["fengxian-city-buildings"]);
|
||||||
expect(CITY_LAYERS.map((layer) => layer.id)).toEqual(["fengxian-city-buildings"]);
|
expect(CITY_LAYERS.map((layer) => layer.id)).toEqual(["fengxian-city-buildings"]);
|
||||||
});
|
});
|
||||||
|
it("uses one absolute archive URL for the PMTiles protocol and persistent cache", () => {
|
||||||
|
expect(cityBuildingArchiveUrl("https://next.waternetwork.cn/base")).toBe(
|
||||||
|
"https://next.waternetwork.cn/city-model/buildings.pmtiles"
|
||||||
|
);
|
||||||
|
});
|
||||||
it("uses a stable directional light for the engineering white model", () => {
|
it("uses a stable directional light for the engineering white model", () => {
|
||||||
expect(CITY_LIGHT).toEqual({
|
expect(CITY_LIGHT).toEqual({
|
||||||
anchor: "map",
|
anchor: "map",
|
||||||
@@ -52,6 +64,18 @@ describe("city scene geographic contract", () => {
|
|||||||
it("accepts WGS84 scene bounds", () => {
|
it("accepts WGS84 scene bounds", () => {
|
||||||
expect(cityManifestSchema.safeParse(manifest).success).toBe(true);
|
expect(cityManifestSchema.safeParse(manifest).success).toBe(true);
|
||||||
});
|
});
|
||||||
|
it("keeps schema v1 scenes visible when cache revision metadata is absent", () => {
|
||||||
|
const { generated_at: _generatedAt, tiles: _tiles, ...legacyManifest } = manifest;
|
||||||
|
expect(cityManifestSchema.safeParse(legacyManifest).success).toBe(true);
|
||||||
|
});
|
||||||
|
it("rejects malformed building archive digests", () => {
|
||||||
|
expect(
|
||||||
|
cityManifestSchema.safeParse({
|
||||||
|
...manifest,
|
||||||
|
tiles: { buildings: { sha256: "not-a-content-digest" } }
|
||||||
|
}).success
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
it("rejects 3857 meters passed as longitude and latitude", () => {
|
it("rejects 3857 meters passed as longitude and latitude", () => {
|
||||||
expect(
|
expect(
|
||||||
cityManifestSchema.safeParse({ ...manifest, bbox: [13508801, 3608163, 13555650, 3633685] })
|
cityManifestSchema.safeParse({ ...manifest, bbox: [13508801, 3608163, 13555650, 3633685] })
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ export const cityManifestSchema = z
|
|||||||
.object({
|
.object({
|
||||||
schema_version: z.literal(1),
|
schema_version: z.literal(1),
|
||||||
name: z.string(),
|
name: z.string(),
|
||||||
|
generated_at: z.string().datetime({ offset: true }).optional(),
|
||||||
bbox: z.tuple([
|
bbox: z.tuple([
|
||||||
z.number().min(-180).max(180),
|
z.number().min(-180).max(180),
|
||||||
z.number().min(-90).max(90),
|
z.number().min(-90).max(90),
|
||||||
@@ -33,14 +34,29 @@ export const cityManifestSchema = z
|
|||||||
pipe_count: z.number().int().nonnegative(),
|
pipe_count: z.number().int().nonnegative(),
|
||||||
asset_count: z.number().int().nonnegative(),
|
asset_count: z.number().int().nonnegative(),
|
||||||
notice: z.string(),
|
notice: z.string(),
|
||||||
attribution: z.string()
|
attribution: z.string(),
|
||||||
|
tiles: z
|
||||||
|
.object({
|
||||||
|
buildings: z.object({
|
||||||
|
sha256: z
|
||||||
|
.string()
|
||||||
|
.regex(/^[a-f0-9]{64}$/)
|
||||||
|
.optional()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.optional()
|
||||||
})
|
})
|
||||||
.refine(({ bbox: [w, s, e, n] }) => w < e && s < n, "Invalid scene bounds");
|
.refine(({ bbox: [w, s, e, n] }) => w < e && s < n, "Invalid scene bounds");
|
||||||
|
|
||||||
|
export function cityBuildingArchiveUrl(origin = window.location.origin) {
|
||||||
|
return new URL(`${CITY_ROOT}/buildings.pmtiles`, origin).href;
|
||||||
|
}
|
||||||
|
|
||||||
export function citySources(): Record<string, SourceSpecification> {
|
export function citySources(): Record<string, SourceSpecification> {
|
||||||
return {
|
return {
|
||||||
[CITY_PREFIX + "buildings"]: {
|
[CITY_PREFIX + "buildings"]: {
|
||||||
type: "vector",
|
type: "vector",
|
||||||
url: `pmtiles://${window.location.origin}${CITY_ROOT}/buildings.pmtiles`,
|
url: `pmtiles://${cityBuildingArchiveUrl()}`,
|
||||||
minzoom: 12,
|
minzoom: 12,
|
||||||
maxzoom: 16,
|
maxzoom: 16,
|
||||||
attribution:
|
attribution:
|
||||||
|
|||||||
@@ -109,6 +109,81 @@ test("city buildings render only from zoom 12 in 3D", async ({ page }) => {
|
|||||||
await expect.poll(renderedBuildings).toBe(0);
|
await expect.poll(renderedBuildings).toBe(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("city building ranges persist across a page reload", async ({ page }) => {
|
||||||
|
test.skip(
|
||||||
|
process.env.TJWATER_CITY_SCENE_TEST !== "1",
|
||||||
|
"Requires the generated local city-model scene"
|
||||||
|
);
|
||||||
|
test.setTimeout(90_000);
|
||||||
|
await mockWorkbenchContext(page, undefined, undefined, {}, [], {
|
||||||
|
tiandituToken: "city-scene-tianditu-token"
|
||||||
|
});
|
||||||
|
await page.route("https://t0.tianditu.gov.cn/**", (route) =>
|
||||||
|
route.fulfill({ contentType: "image/png", body: EMPTY_RASTER_TILE })
|
||||||
|
);
|
||||||
|
|
||||||
|
const buildingRequests: string[] = [];
|
||||||
|
page.on("request", (request) => {
|
||||||
|
if (new URL(request.url()).pathname === "/city-model/buildings.pmtiles") {
|
||||||
|
buildingRequests.push(request.headers().range ?? "");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const openScene = async () => {
|
||||||
|
await page.goto("/", { waitUntil: "domcontentloaded" });
|
||||||
|
const toggle = page
|
||||||
|
.getByRole("group", { name: "地图缩放", exact: true })
|
||||||
|
.locator("button[aria-pressed]");
|
||||||
|
await expect(toggle).toBeEnabled({ timeout: 15_000 });
|
||||||
|
await toggle.click();
|
||||||
|
await page.evaluate(() =>
|
||||||
|
window.__waterNetworkMap!.jumpTo({
|
||||||
|
center: [121.464, 30.916],
|
||||||
|
zoom: 14,
|
||||||
|
pitch: 55,
|
||||||
|
bearing: 0
|
||||||
|
})
|
||||||
|
);
|
||||||
|
await expect
|
||||||
|
.poll(
|
||||||
|
() =>
|
||||||
|
page.evaluate(
|
||||||
|
() =>
|
||||||
|
window.__waterNetworkMap?.queryRenderedFeatures({
|
||||||
|
layers: ["fengxian-city-buildings"]
|
||||||
|
}).length ?? 0
|
||||||
|
),
|
||||||
|
{ timeout: 30_000 }
|
||||||
|
)
|
||||||
|
.toBeGreaterThan(0);
|
||||||
|
await waitForMapIdle(page);
|
||||||
|
};
|
||||||
|
|
||||||
|
await openScene();
|
||||||
|
expect(buildingRequests.length).toBeGreaterThan(0);
|
||||||
|
expect(buildingRequests.every((range) => /^bytes=\d+-\d+$/.test(range))).toBe(true);
|
||||||
|
|
||||||
|
const cachedRangeCount = await page.evaluate(
|
||||||
|
() =>
|
||||||
|
new Promise<number>((resolve, reject) => {
|
||||||
|
const request = indexedDB.open("tjwater-city-pmtiles-cache");
|
||||||
|
request.onerror = () => reject(request.error);
|
||||||
|
request.onsuccess = () => {
|
||||||
|
const database = request.result;
|
||||||
|
const transaction = database.transaction("ranges", "readonly");
|
||||||
|
const countRequest = transaction.objectStore("ranges").count();
|
||||||
|
countRequest.onerror = () => reject(countRequest.error);
|
||||||
|
countRequest.onsuccess = () => resolve(countRequest.result);
|
||||||
|
};
|
||||||
|
})
|
||||||
|
);
|
||||||
|
expect(cachedRangeCount).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
buildingRequests.length = 0;
|
||||||
|
await openScene();
|
||||||
|
expect(buildingRequests).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
// Uses generated local building data, while unrelated backend services are isolated.
|
// Uses generated local building data, while unrelated backend services are isolated.
|
||||||
test("3D adds opaque buildings without enabling duplicate GIS layers", async ({ page }) => {
|
test("3D adds opaque buildings without enabling duplicate GIS layers", async ({ page }) => {
|
||||||
test.skip(
|
test.skip(
|
||||||
|
|||||||
Reference in New Issue
Block a user