Benchmarks v3 migration to duckdb (#7646)

This is a one-shot migration binary to take all of the data from
`data.json.gz` and bring it into a duckdb database.

Simply gathers and aggregates everything into memory and writes data in
chunks with arrow arrays. Insert row-by-row took way too long, and the
appender API in duckdb does not support `BIGINT[]` for some reason...

---------

Signed-off-by: Claude <noreply@anthropic.com>
Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>

Author: connortsui20

Cargo.lock

@@ -62,6 +62,7 @@ members = [
     "benchmarks/vector-search-bench",
     # Benchmarks website v3 (alpha) - leaf binary, not part of vortex-* API
     "benchmarks-website/server",
+    "benchmarks-website/migrate",
 ]
 exclude = ["java/testfiles", "wasm-test"]
 resolver = "2"

Cargo.toml

@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+[package]
+name = "vortex-bench-migrate"
+version = "0.1.0-alpha.0"
+edition = "2024"
+rust-version = "1.91.0"
+license = "Apache-2.0"
+description = "One-shot historical migrator from the v2 benchmarks S3 dataset to a v3 DuckDB file"
+publish = false
+
+[[bin]]
+name = "vortex-bench-migrate"
+path = "src/main.rs"
+
+# Throwaway binary, not part of the vortex-* public API surface.
+# Errors use anyhow, and the crate is intentionally outside the
+# workspace public-api lockfile set.
+
+[dependencies]
+anyhow = { workspace = true }
+arrow-array = { workspace = true }
+arrow-buffer = { workspace = true }
+arrow-schema = { workspace = true }
+clap = { workspace = true, features = ["derive"] }
+# track vortex-duckdb's bundled engine version (build.rs)
+duckdb = { version = "1.10502", features = ["bundled", "appender-arrow"] }
+flate2 = "1.1"
+reqwest = { workspace = true, features = ["json"] }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
+tracing = { workspace = true, features = ["std"] }
+tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] }
+vortex-bench-server = { path = "../server" }
+vortex-utils = { workspace = true }
+
+[dev-dependencies]
+rstest = { workspace = true }
+tempfile = { workspace = true }