From 8e7c7d4cf518b961a9f533976ddccf3d43b6214e Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 19 Dec 2024 16:43:56 -0500 Subject: [PATCH 1/4] Parse WKB to GeoArrow --- package.json | 4 + src/algorithm/utils/assert.ts | 4 +- src/data.ts | 2 + src/io/wkb.ts | 270 ++++++++++++++++++++++++++++++++++ src/type.ts | 2 + yarn.lock | 172 ++++++++++++++++++++++ 6 files changed, 452 insertions(+), 2 deletions(-) create mode 100644 src/io/wkb.ts diff --git a/package.json b/package.json index f178416..a735530 100644 --- a/package.json +++ b/package.json @@ -44,9 +44,13 @@ "src/" ], "peerDependencies": { + "@loaders.gl/schema": "*", + "@loaders.gl/wkt": "*", "apache-arrow": ">=15" }, "devDependencies": { + "@loaders.gl/schema": "^4.3.3", + "@loaders.gl/wkt": "^4.3.3", "@rollup/plugin-node-resolve": "^15.2.3", "@rollup/plugin-terser": "^0.4.3", "@rollup/plugin-typescript": "^11.1.2", diff --git a/src/algorithm/utils/assert.ts b/src/algorithm/utils/assert.ts index 522f905..6db7bce 100644 --- a/src/algorithm/utils/assert.ts +++ b/src/algorithm/utils/assert.ts @@ -4,6 +4,6 @@ export function assert(condition: boolean, message?: string) { } } -export function assertFalse(): never { - throw new Error(`assertion failed`); +export function assertFalse(message?: string): never { + throw new Error(`assertion failed ${message}`); } diff --git a/src/data.ts b/src/data.ts index f37f156..57b5e6f 100644 --- a/src/data.ts +++ b/src/data.ts @@ -1,5 +1,6 @@ import { Data } from "apache-arrow/data"; import { + WKB, Point, LineString, Polygon, @@ -27,6 +28,7 @@ export type GeoArrowData = | MultiPointData | MultiLineStringData | MultiPolygonData; +export type WKBData = Data; export function isPointData(data: Data): data is PointData { return isPoint(data.type); diff --git a/src/io/wkb.ts b/src/io/wkb.ts new file mode 100644 index 0000000..dde985a --- /dev/null +++ b/src/io/wkb.ts @@ -0,0 +1,270 @@ +import { makeData } from "apache-arrow/data"; +import { + GeoArrowData, + LineStringData, + PointData, + PolygonData, + WKBData, +} from "../data"; +import { WKBLoader } from "@loaders.gl/wkt"; +import type { + BinaryGeometry, + BinaryPointGeometry, + BinaryLineGeometry, + BinaryPolygonGeometry, +} from "@loaders.gl/schema"; +import { assert, assertFalse } from "../algorithm/utils/assert"; +import { Field, FixedSizeList, Float64, List } from "apache-arrow"; + +export enum WKBType { + Point, + LineString, + Polygon, + MultiPoint, + MultiLineString, + MultiPolygon, +} + +/** + * Parse an Arrow array of WKB + * + * @return {[type]} [return description] + */ +export function parseWkb( + data: WKBData, + type: WKBType, + dim: number, +): GeoArrowData { + const parsedGeometries: BinaryGeometry[] = []; + + for (const item of iterBinary(data)) { + if (item === null) { + throw new Error("Null entries are not currently supported"); + } + const arrayBuffer = copyViewToArrayBuffer(item); + const parsed = WKBLoader.parseSync(arrayBuffer, { + wkb: { shape: "binary-geometry" }, + }) as BinaryGeometry; + parsedGeometries.push(parsed); + } + + switch (type) { + case WKBType.Point: + return repackPoints(parsedGeometries as BinaryPointGeometry[], dim); + + case WKBType.LineString: + return repackLineStrings(parsedGeometries as BinaryLineGeometry[], dim); + + case WKBType.Polygon: + return repackPolygons(parsedGeometries as BinaryPolygonGeometry[], dim); + + default: + assertFalse("Not yet implemented for this geometry type"); + } +} + +function* iterBinary(data: WKBData): IterableIterator { + const values = data.values; + const valueOffsets = data.valueOffsets; + for (let i = 0; i < data.length; i++) { + if (!data.getValid(i)) { + yield null; + } else { + const startOffset = valueOffsets[i]; + const endOffset = valueOffsets[i + 1]; + yield values.subarray(startOffset, endOffset); + } + } +} + +// TODO: update loaders.gl parseWKB to accept Uint8Array, not just ArrayBuffer. +function copyViewToArrayBuffer(view: Uint8Array): ArrayBuffer { + const buffer = new ArrayBuffer(view.byteLength); + new Uint8Array(buffer).set(view); + return buffer; +} + +function repackPoints(geoms: BinaryPointGeometry[], dim: number): PointData { + const geomLength = geoms.length; + const coords = new Float64Array(geomLength * dim); + let coordOffset = 0; + for (const geom of geoms) { + assert(geom.positions.value instanceof Float64Array); + coords.set(geom.positions.value, coordOffset * dim); + coordOffset += 1; + } + + const coordsData = makeData({ + type: new Float64(), + data: coords, + }); + return makeData({ + type: new FixedSizeList( + dim, + new Field(coordFieldName(dim), new Float64(), false), + ), + child: coordsData, + }); +} + +type LineStringCapacity = { + coordCapacty: number; + geomCapacity: number; +}; + +function repackLineStrings( + geoms: BinaryLineGeometry[], + dim: number, +): LineStringData { + const capacity = inferLineStringCapacity(geoms); + const coords = new Float64Array(capacity.coordCapacty * dim); + const geomOffsets = new Int32Array(capacity.geomCapacity + 1); + + let geomIndex = 0; + let coordOffset = 0; + for (const geom of geoms) { + assert(geom.positions.value instanceof Float64Array); + const numCoords = geom.positions.value.length / geom.positions.size; + coords.set(geom.positions.value, coordOffset * dim); + geomIndex += 1; + coordOffset += numCoords; + + // Note this is after we've added one + geomOffsets[geomIndex] = coordOffset; + } + + const coordsData = makeData({ + type: new Float64(), + data: coords, + }); + const verticesData = makeData({ + type: new FixedSizeList( + dim, + new Field(coordFieldName(dim), coordsData.type, false), + ), + child: coordsData, + }); + return makeData({ + type: new List(new Field("vertices", verticesData.type, false)), + valueOffsets: geomOffsets, + child: verticesData, + }); +} + +function inferLineStringCapacity( + geoms: BinaryLineGeometry[], +): LineStringCapacity { + let capacity: LineStringCapacity = { + coordCapacty: 0, + geomCapacity: 0, + }; + + // TODO: check geom.pathIndices to validate that we have a LineString and not + // a multi line string + for (const geom of geoms) { + capacity.geomCapacity += 1; + capacity.coordCapacty += geom.positions.value.length / geom.positions.size; + } + + return capacity; +} + +type PolygonCapacity = { + coordCapacty: number; + /** This is what loaders.gl calls `primitivePolygonIndices` */ + ringCapacity: number; + geomCapacity: number; +}; + +function repackPolygons( + geoms: BinaryPolygonGeometry[], + dim: number, +): PolygonData { + const capacity = inferPolygonCapacity(geoms); + const coords = new Float64Array(capacity.coordCapacty * dim); + const ringOffsets = new Int32Array(capacity.ringCapacity + 1); + const geomOffsets = new Int32Array(capacity.geomCapacity + 1); + + let geomIndex = 0; + let coordOffset = 0; + let ringOffset = 0; + + for (const geom of geoms) { + assert(geom.positions.value instanceof Float64Array); + const numCoords = geom.positions.value.length / geom.positions.size; + + coords.set(geom.positions.value, coordOffset * dim); + coordOffset += numCoords; + + for ( + let ringIdx = 0; + ringIdx < geom.primitivePolygonIndices.value.length - 1; + ringIdx++ + ) { + ringOffsets[ringOffset + 1] = + (geom.primitivePolygonIndices.value[ringOffset + 1] - + geom.primitivePolygonIndices.value[ringOffset]) / + geom.positions.size; + ringOffset += 1; + } + + geomOffsets[geomIndex + 1] = ringOffset; + geomIndex += 1; + } + + const coordsData = makeData({ + type: new Float64(), + data: coords, + }); + const verticesData = makeData({ + type: new FixedSizeList( + dim, + new Field(coordFieldName(dim), coordsData.type, false), + ), + child: coordsData, + }); + + const ringsData = makeData({ + type: new List(new Field("vertices", verticesData.type, false)), + valueOffsets: ringOffsets, + child: verticesData, + }); + + return makeData({ + type: new List(new Field("rings", ringsData.type, false)), + valueOffsets: geomOffsets, + child: ringsData, + }); +} + +function inferPolygonCapacity(geoms: BinaryPolygonGeometry[]): PolygonCapacity { + let capacity: PolygonCapacity = { + coordCapacty: 0, + ringCapacity: 0, + geomCapacity: 0, + }; + + // TODO: check geom.polygonIndices to validate that we have a Polygon and not + // a MultiPolygon + for (const geom of geoms) { + capacity.geomCapacity += 1; + assert( + geom.primitivePolygonIndices.value.length >= 1, + "Expected primitivePolygonIndices to always have length at least 1", + ); + capacity.ringCapacity += geom.primitivePolygonIndices.value.length - 1; + capacity.coordCapacty += geom.positions.value.length / geom.positions.size; + } + + return capacity; +} + +function coordFieldName(dim: number): "xy" | "xyz" { + if (dim === 2) { + return "xy"; + } else if (dim === 3) { + return "xyz"; + } else { + assertFalse("expected dimension of 2 or 3"); + } +} diff --git a/src/type.ts b/src/type.ts index 5796ae9..285822b 100644 --- a/src/type.ts +++ b/src/type.ts @@ -1,4 +1,5 @@ import { + Binary, Struct, Float, List, @@ -29,6 +30,7 @@ export type GeoArrowType = | MultiPoint | MultiLineString | MultiPolygon; +export type WKB = Binary; /** Check that the given type is a Point data type */ export function isPoint(type: DataType): type is Point { diff --git a/yarn.lock b/yarn.lock index 6956a49..2b9157d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -226,6 +226,8 @@ __metadata: version: 0.0.0-use.local resolution: "@geoarrow/geoarrow-js@workspace:." dependencies: + "@loaders.gl/schema": "npm:^4.3.3" + "@loaders.gl/wkt": "npm:^4.3.3" "@math.gl/polygon": "npm:^4.0.0" "@rollup/plugin-node-resolve": "npm:^15.2.3" "@rollup/plugin-terser": "npm:^0.4.3" @@ -246,6 +248,8 @@ __metadata: typescript: "npm:^5.2.2" vitest: "npm:^0.34.6" peerDependencies: + "@loaders.gl/schema": "*" + "@loaders.gl/wkt": "*" apache-arrow: ">=15" languageName: unknown linkType: soft @@ -335,6 +339,84 @@ __metadata: languageName: node linkType: hard +"@loaders.gl/gis@npm:4.3.3": + version: 4.3.3 + resolution: "@loaders.gl/gis@npm:4.3.3" + dependencies: + "@loaders.gl/loader-utils": "npm:4.3.3" + "@loaders.gl/schema": "npm:4.3.3" + "@mapbox/vector-tile": "npm:^1.3.1" + "@math.gl/polygon": "npm:^4.1.0" + pbf: "npm:^3.2.1" + peerDependencies: + "@loaders.gl/core": ^4.3.0 + checksum: 35ee958d164e894c510dd43caaf630f6007568bc36523a2c2e2744c86e8008e039c542d3693d6655fc7db3b510058e8984da01fbb15b905f5c6e299830cb1c13 + languageName: node + linkType: hard + +"@loaders.gl/loader-utils@npm:4.3.3": + version: 4.3.3 + resolution: "@loaders.gl/loader-utils@npm:4.3.3" + dependencies: + "@loaders.gl/schema": "npm:4.3.3" + "@loaders.gl/worker-utils": "npm:4.3.3" + "@probe.gl/log": "npm:^4.0.2" + "@probe.gl/stats": "npm:^4.0.2" + peerDependencies: + "@loaders.gl/core": ^4.3.0 + checksum: b95417430528af7e2fda748406643d2f5e709fd05216b4959d774e4a12fff56155bed28cc1847e1af2761070954367a3eed8f5ebe5ca4c13f26a7cedc30de1e0 + languageName: node + linkType: hard + +"@loaders.gl/schema@npm:4.3.3, @loaders.gl/schema@npm:^4.3.3": + version: 4.3.3 + resolution: "@loaders.gl/schema@npm:4.3.3" + dependencies: + "@types/geojson": "npm:^7946.0.7" + peerDependencies: + "@loaders.gl/core": ^4.3.0 + checksum: a04888ec04d0f44aed72c5c4130df6a7aca6cb872cffcc5616f79478d808789580a3292c8fa778761f99f45b1581224a62095c20f690a57782f24124e685520e + languageName: node + linkType: hard + +"@loaders.gl/wkt@npm:^4.3.3": + version: 4.3.3 + resolution: "@loaders.gl/wkt@npm:4.3.3" + dependencies: + "@loaders.gl/gis": "npm:4.3.3" + "@loaders.gl/loader-utils": "npm:4.3.3" + "@loaders.gl/schema": "npm:4.3.3" + peerDependencies: + "@loaders.gl/core": ^4.3.0 + checksum: 9fd227b72b41811bc18775147f96069b5a6036e830826a3b3378c635415a41e733ff866ee627bc48771e3c08303c354ea23ac5e8c0a6a35dbd712fc1e769034e + languageName: node + linkType: hard + +"@loaders.gl/worker-utils@npm:4.3.3": + version: 4.3.3 + resolution: "@loaders.gl/worker-utils@npm:4.3.3" + peerDependencies: + "@loaders.gl/core": ^4.3.0 + checksum: e0fea94518a226ee2ecbcf40ae76eab2c0b6fdca3dc596b03b9febc11f0d66cec1c5c3d1c6908f47f87b3b950f2a4b3ad04be4012e26119ce838953bff37a2e7 + languageName: node + linkType: hard + +"@mapbox/point-geometry@npm:~0.1.0": + version: 0.1.0 + resolution: "@mapbox/point-geometry@npm:0.1.0" + checksum: f6f78ac8a7f798efb19db6eb1a9e05da7ba942102f5347c1a673d94202d0c606ec3f522efa3e76d583cdca46fb96dde52c3d37234f162d21df42f9e8c4f182bd + languageName: node + linkType: hard + +"@mapbox/vector-tile@npm:^1.3.1": + version: 1.3.1 + resolution: "@mapbox/vector-tile@npm:1.3.1" + dependencies: + "@mapbox/point-geometry": "npm:~0.1.0" + checksum: ed31eeef0d593befde76b5b4edf0472709a2ba66dd6b32fad5671caa245fdac976e23ff385facf36e297f14a53c905bfde8911599e8aa690354d52b22bc4cfc5 + languageName: node + linkType: hard + "@math.gl/core@npm:4.0.0": version: 4.0.0 resolution: "@math.gl/core@npm:4.0.0" @@ -345,6 +427,15 @@ __metadata: languageName: node linkType: hard +"@math.gl/core@npm:4.1.0": + version: 4.1.0 + resolution: "@math.gl/core@npm:4.1.0" + dependencies: + "@math.gl/types": "npm:4.1.0" + checksum: 4b4c1c4cd08df24687e90469f2e17ad4db9547df58e45585a1f91fb9590b6f96ef3287fbc7f4b9aabc9531d4c223748354a3633ea0ed354c3ff96e11243f19f9 + languageName: node + linkType: hard + "@math.gl/polygon@npm:^4.0.0": version: 4.0.0 resolution: "@math.gl/polygon@npm:4.0.0" @@ -354,6 +445,15 @@ __metadata: languageName: node linkType: hard +"@math.gl/polygon@npm:^4.1.0": + version: 4.1.0 + resolution: "@math.gl/polygon@npm:4.1.0" + dependencies: + "@math.gl/core": "npm:4.1.0" + checksum: ea2b2f9ec9772f04b9f57691d472374d149960287deb33f91a87e28aafa81a95a3ff1101bc59d51be47a2aade79275e7fe0df6b4cd8d04a06c652ea93365efbc + languageName: node + linkType: hard + "@math.gl/types@npm:4.0.0": version: 4.0.0 resolution: "@math.gl/types@npm:4.0.0" @@ -361,6 +461,13 @@ __metadata: languageName: node linkType: hard +"@math.gl/types@npm:4.1.0": + version: 4.1.0 + resolution: "@math.gl/types@npm:4.1.0" + checksum: 55ae7f4e2b89832c417a59cfb93f3a1fe37fda89d84678562a06fbae19e082e6e4147383963b7e67d8507bc5979c9347b7ce7598ec8daca4c4fee48ae00cbece + languageName: node + linkType: hard + "@npmcli/agent@npm:^2.0.0": version: 2.2.1 resolution: "@npmcli/agent@npm:2.2.1" @@ -390,6 +497,29 @@ __metadata: languageName: node linkType: hard +"@probe.gl/env@npm:4.0.9": + version: 4.0.9 + resolution: "@probe.gl/env@npm:4.0.9" + checksum: ea8a98619375f53867f0d5bdf0f59b89f9862c4185a9c742f922278d5eeb78578ec4a3a2d729aa3a1c9cafa167fbc3981bc03dc3034a40bd4ade6491ffad4f3b + languageName: node + linkType: hard + +"@probe.gl/log@npm:^4.0.2": + version: 4.0.9 + resolution: "@probe.gl/log@npm:4.0.9" + dependencies: + "@probe.gl/env": "npm:4.0.9" + checksum: 650667b35cf075507c23725b1dcda8bf4fb5d562bcdc6fa6305defa221aefeca3a1975cd893db0367a45a8419418f258e6becc6c7f33430b7fa48a78c9b28cd9 + languageName: node + linkType: hard + +"@probe.gl/stats@npm:^4.0.2": + version: 4.0.9 + resolution: "@probe.gl/stats@npm:4.0.9" + checksum: 61c31bdc523952e65561f3be3ef34f537c94593cafa95124fd25540a06046d6b14076c8d539806ee53f24101cf8b6e3fd461b4d1d2aee266ec915642c787b379 + languageName: node + linkType: hard + "@rollup/plugin-node-resolve@npm:^15.2.3": version: 15.2.3 resolution: "@rollup/plugin-node-resolve@npm:15.2.3" @@ -632,6 +762,13 @@ __metadata: languageName: node linkType: hard +"@types/geojson@npm:^7946.0.7": + version: 7946.0.15 + resolution: "@types/geojson@npm:7946.0.15" + checksum: 226d7ab59540632b19f7889c76c4c586a5104c18c43a81f32974aa035eafe557f86bd5a79ca5568bb63cbe5bfa9014c8e9a29cb0bb3d2f0bd71b0cc13ad8ccb3 + languageName: node + linkType: hard + "@types/node@npm:*, @types/node@npm:^20.6.0, @types/node@npm:^20.9.3": version: 20.11.24 resolution: "@types/node@npm:20.11.24" @@ -1578,6 +1715,13 @@ __metadata: languageName: node linkType: hard +"ieee754@npm:^1.1.12": + version: 1.2.1 + resolution: "ieee754@npm:1.2.1" + checksum: d9f2557a59036f16c282aaeb107832dc957a93d73397d89bbad4eb1130560560eb695060145e8e6b3b498b15ab95510226649a0b8f52ae06583575419fe10fc4 + languageName: node + linkType: hard + "imurmurhash@npm:^0.1.4": version: 0.1.4 resolution: "imurmurhash@npm:0.1.4" @@ -2150,6 +2294,18 @@ __metadata: languageName: node linkType: hard +"pbf@npm:^3.2.1": + version: 3.3.0 + resolution: "pbf@npm:3.3.0" + dependencies: + ieee754: "npm:^1.1.12" + resolve-protobuf-schema: "npm:^2.1.0" + bin: + pbf: bin/pbf + checksum: 46488694528740097c33443efa240ca7f99538a2b96e9fbd2284d9be45ec91dab6954b9c03df237656ac2757d7f046153a031ad24519b4cfcaa2e7b18ddeb5dd + languageName: node + linkType: hard + "picocolors@npm:^1.0.0": version: 1.0.0 resolution: "picocolors@npm:1.0.0" @@ -2265,6 +2421,13 @@ __metadata: languageName: node linkType: hard +"protocol-buffers-schema@npm:^3.3.1": + version: 3.6.0 + resolution: "protocol-buffers-schema@npm:3.6.0" + checksum: 55a1caed123fb2385eae5ea4770dc36b3017d1fe2005ffb1ef20c97dadf43a91876238ebc23bc240ef1f8501d054bdd9d12992796e9abed18ddf958e4f942eea + languageName: node + linkType: hard + "randombytes@npm:^2.1.0": version: 2.1.0 resolution: "randombytes@npm:2.1.0" @@ -2288,6 +2451,15 @@ __metadata: languageName: node linkType: hard +"resolve-protobuf-schema@npm:^2.1.0": + version: 2.1.0 + resolution: "resolve-protobuf-schema@npm:2.1.0" + dependencies: + protocol-buffers-schema: "npm:^3.3.1" + checksum: 88fffab2a3757888884a36f9aa4e24be5186b01820a8c26297dc1ce406b9daf776594926bdf524c2c8e8e5b0aba8ac48362b6584cdecc9a7083215ebca01c599 + languageName: node + linkType: hard + "resolve@npm:^1.22.1": version: 1.22.8 resolution: "resolve@npm:1.22.8" From 72c600c1729d8d515a27172a60fd4e1082d4454e Mon Sep 17 00:00:00 2001 From: Saadiq Mohiuddin <34844565+smohiudd@users.noreply.github.com> Date: Fri, 20 Dec 2024 14:31:01 -0700 Subject: [PATCH 2/4] update type imports (#30) --- src/index.ts | 1 + src/io/index.ts | 1 + src/io/wkb.ts | 5 ++++- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 src/io/index.ts diff --git a/src/index.ts b/src/index.ts index a476be7..88fa4a6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,3 +4,4 @@ export * as data from "./data.js"; export * as type from "./type.js"; export * as vector from "./vector.js"; export * as worker from "./worker"; +export * as io from "./io"; diff --git a/src/io/index.ts b/src/io/index.ts new file mode 100644 index 0000000..73e6e2a --- /dev/null +++ b/src/io/index.ts @@ -0,0 +1 @@ +export { parseWkb, WKBType } from "./wkb.js"; diff --git a/src/io/wkb.ts b/src/io/wkb.ts index dde985a..c4444f9 100644 --- a/src/io/wkb.ts +++ b/src/io/wkb.ts @@ -14,7 +14,9 @@ import type { BinaryPolygonGeometry, } from "@loaders.gl/schema"; import { assert, assertFalse } from "../algorithm/utils/assert"; -import { Field, FixedSizeList, Float64, List } from "apache-arrow"; +import { FixedSizeList, Float64, List } from "apache-arrow/type"; +import { Field } from "apache-arrow/schema"; + export enum WKBType { Point, @@ -37,6 +39,7 @@ export function parseWkb( ): GeoArrowData { const parsedGeometries: BinaryGeometry[] = []; + for (const item of iterBinary(data)) { if (item === null) { throw new Error("Null entries are not currently supported"); From f1d6c936a18043993754d09badbeb1e33825848a Mon Sep 17 00:00:00 2001 From: Hanbyul Jo Date: Mon, 6 Jan 2025 17:47:42 -0500 Subject: [PATCH 3/4] Consolidate apache arrow imports (#33) * Import from apache arrow root level * Fix lint * Consolidate imports to use the main one --- src/child.ts | 4 +--- src/data.ts | 2 +- src/io/wkb.ts | 6 +----- src/type.ts | 12 ++++++------ src/vector.ts | 2 +- src/worker/hard-clone.ts | 5 +---- src/worker/rehydrate.ts | 13 +++++++------ src/worker/transferable.ts | 6 ++---- 8 files changed, 20 insertions(+), 30 deletions(-) diff --git a/src/child.ts b/src/child.ts index 3216d50..c174b93 100644 --- a/src/child.ts +++ b/src/child.ts @@ -2,9 +2,7 @@ * Strongly typed accessors for children, since arrow.Data.children[] is untyped */ -import { Data } from "apache-arrow/data"; -import { Vector } from "apache-arrow/vector"; -import { Float } from "apache-arrow/type"; +import { Data, Vector, type Float } from "apache-arrow"; import { LineStringData, MultiLineStringData, diff --git a/src/data.ts b/src/data.ts index 57b5e6f..03d74dd 100644 --- a/src/data.ts +++ b/src/data.ts @@ -1,4 +1,4 @@ -import { Data } from "apache-arrow/data"; +import { Data } from "apache-arrow"; import { WKB, Point, diff --git a/src/io/wkb.ts b/src/io/wkb.ts index c4444f9..e6cbb52 100644 --- a/src/io/wkb.ts +++ b/src/io/wkb.ts @@ -1,4 +1,4 @@ -import { makeData } from "apache-arrow/data"; +import { makeData, Field, FixedSizeList, Float64, List } from "apache-arrow"; import { GeoArrowData, LineStringData, @@ -14,9 +14,6 @@ import type { BinaryPolygonGeometry, } from "@loaders.gl/schema"; import { assert, assertFalse } from "../algorithm/utils/assert"; -import { FixedSizeList, Float64, List } from "apache-arrow/type"; -import { Field } from "apache-arrow/schema"; - export enum WKBType { Point, @@ -39,7 +36,6 @@ export function parseWkb( ): GeoArrowData { const parsedGeometries: BinaryGeometry[] = []; - for (const item of iterBinary(data)) { if (item === null) { throw new Error("Null entries are not currently supported"); diff --git a/src/type.ts b/src/type.ts index 285822b..8cd8d21 100644 --- a/src/type.ts +++ b/src/type.ts @@ -1,11 +1,11 @@ import { - Binary, - Struct, - Float, - List, - FixedSizeList, + type Binary, + type Struct, + type Float, + type List, + type FixedSizeList, DataType, -} from "apache-arrow/type"; +} from "apache-arrow"; // Note: this apparently has to be arrow.Float and not arrow.Float64 to ensure // that recreating a data instance with arrow.makeData type checks using the diff --git a/src/vector.ts b/src/vector.ts index 54da13a..80bf4d1 100644 --- a/src/vector.ts +++ b/src/vector.ts @@ -1,4 +1,4 @@ -import { Vector } from "apache-arrow/vector"; +import { Vector } from "apache-arrow"; import { Point, LineString, diff --git a/src/worker/hard-clone.ts b/src/worker/hard-clone.ts index edf9f65..566d59e 100644 --- a/src/worker/hard-clone.ts +++ b/src/worker/hard-clone.ts @@ -1,7 +1,4 @@ -import { Data } from "apache-arrow/data"; -import { DataType } from "apache-arrow/type"; -import { Vector } from "apache-arrow/vector"; -import { BufferType } from "apache-arrow/enum"; +import { Data, DataType, Vector, BufferType } from "apache-arrow"; import type { Buffers } from "apache-arrow/data"; type TypedArray = diff --git a/src/worker/rehydrate.ts b/src/worker/rehydrate.ts index 5bf25b0..c878566 100644 --- a/src/worker/rehydrate.ts +++ b/src/worker/rehydrate.ts @@ -1,5 +1,9 @@ import { - DataType, + BufferType, + Type, + Data, + Vector, + Field, Null, Int, Float, @@ -18,11 +22,8 @@ import { FixedSizeList, Map_, Duration, -} from "apache-arrow/type"; -import { BufferType, Type } from "apache-arrow/enum"; -import { Data } from "apache-arrow/data"; -import { Vector } from "apache-arrow/vector"; -import { Field } from "apache-arrow/schema"; + type DataType, +} from "apache-arrow"; import type { Buffers } from "apache-arrow/data"; import { Polygon, isPolygon } from "../type"; import { PolygonData } from "../data"; diff --git a/src/worker/transferable.ts b/src/worker/transferable.ts index ff51eaf..80de0ed 100644 --- a/src/worker/transferable.ts +++ b/src/worker/transferable.ts @@ -1,7 +1,4 @@ -import { DataType } from "apache-arrow/type"; -import { BufferType } from "apache-arrow/enum"; -import { Data } from "apache-arrow/data"; -import { Vector } from "apache-arrow/vector"; +import { Data, Vector, BufferType, type DataType } from "apache-arrow"; import { hardClone } from "./hard-clone"; /** @@ -63,6 +60,7 @@ export function preparePostMessage( if (input.buffers[BufferType.OFFSET] !== undefined) { transferArrayBuffers.push(input.buffers[BufferType.OFFSET].buffer); } + if (input.buffers[BufferType.DATA] !== undefined) { transferArrayBuffers.push(input.buffers[BufferType.DATA].buffer); } From a27dba7d88964204ead32b07647f795ef2e00f6c Mon Sep 17 00:00:00 2001 From: Saadiq Mohiuddin <34844565+smohiudd@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:27:32 -0700 Subject: [PATCH 4/4] fix polygon ringoffset (#32) --- src/io/wkb.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/io/wkb.ts b/src/io/wkb.ts index e6cbb52..23e2a53 100644 --- a/src/io/wkb.ts +++ b/src/io/wkb.ts @@ -201,9 +201,9 @@ function repackPolygons( ringIdx++ ) { ringOffsets[ringOffset + 1] = - (geom.primitivePolygonIndices.value[ringOffset + 1] - - geom.primitivePolygonIndices.value[ringOffset]) / - geom.positions.size; + ringOffsets[ringOffset] + + (geom.primitivePolygonIndices.value[ringIdx + 1] - + geom.primitivePolygonIndices.value[ringIdx]); ringOffset += 1; }