diff --git a/Cargo.lock b/Cargo.lock index 509e14e5f..873c64925 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1109,8 +1109,12 @@ dependencies = [ "admin-client", "amp-object-store", "anyhow", + "async-trait", "clap", + "common", "console", + "datafusion", + "dataset-authoring", "datasets-common", "datasets-derived", "dump", @@ -4068,6 +4072,31 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "dataset-authoring" +version = "0.1.0" +dependencies = [ + "async-trait", + "common", + "datafusion", + "datasets-common", + "datasets-derived", + "dirs", + "flate2", + "minijinja", + "regex", + "semver 1.0.27", + "serde", + "serde_json", + "serde_json_canonicalizer 0.2.0", + "serde_yaml", + "tar", + "tempfile", + "thiserror 2.0.18", + "tokio", + "walkdir", +] + [[package]] name = "datasets-common" version = "0.1.0" @@ -4898,6 +4927,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -6764,6 +6804,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minijinja" +version = "2.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b479616bb6f0779fb0f3964246beda02d4b01144e1b0d5519616e012ccc2a245" +dependencies = [ + "serde", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -8033,7 +8082,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "serde_json_canonicalizer", + "serde_json_canonicalizer 0.3.1", "serde_yaml", "sha2", "spki", @@ -9081,6 +9130,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" +[[package]] +name = "ryu-js" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6518fc26bced4d53678a22d6e423e9d8716377def84545fe328236e3af070e7f" + [[package]] name = "ryu-js" version = "1.0.2" @@ -9369,13 +9424,24 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_json_canonicalizer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a4c2ea923e1d3d2c03bd1299b7061e015f3bd9dab5a47b34283df9d8ab36a1" +dependencies = [ + "ryu-js 0.2.2", + "serde", + "serde_json", +] + [[package]] name = "serde_json_canonicalizer" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f777f77aeef456e47e75c2a4b16804b15395be5b344e2094a54965143ef1c31" dependencies = [ - "ryu-js", + "ryu-js 1.0.2", "serde", "serde_json", ] @@ -12161,6 +12227,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.24.0" @@ -12260,12 +12337,15 @@ dependencies = [ "amp-providers-registry", "ampctl", "arrow-flight", + "async-trait", "backon", "clap", "color-backtrace", "common", "controller", "ctor", + "datafusion", + "dataset-authoring", "datasets-common", "datasets-derived", "dotenvy", @@ -14090,6 +14170,16 @@ dependencies = [ "time", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.3", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 0c68b4ade..fd39271ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "crates/clients/flight", "crates/core/common", "crates/core/data-store", + "crates/core/dataset-authoring", "crates/core/dataset-store", "crates/core/datasets-registry", "crates/core/providers-registry", @@ -95,6 +96,8 @@ governor = "0.10.0" hex = "0.4.3" indoc = "2.0.6" itertools = "0.14.0" +minijinja = "2.7" +serde_json_canonicalizer = "0.2" opentelemetry = { version = "0.31", features = ["trace"] } opentelemetry-otlp = { version = "0.31", features = [ "grpc-tonic", @@ -140,12 +143,16 @@ uuid = { version = "1.11.0", features = ["v7"] } nix = { version = "0.30.1", default-features = false, features = ["signal"] } percent-encoding = "2.3" which = "8.0.0" +walkdir = "2" zstd = "0.13.3" +tar = "0.4" +flate2 = "1.1" # Datafusion and Arrow crates arrow = "57" arrow-flight = { version = "57", features = ["flight-sql-experimental"] } datafusion = { version = "52", features = ["serde"] } +dirs = "6.0" datafusion-tracing = { version = "52" } datafusion-datasource = { version = "52" } object_store = { version = "0.12", features = ["aws", "gcp", "azure"] } diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 000000000..326afdb0f --- /dev/null +++ b/SPEC.md @@ -0,0 +1,367 @@ +# Dataset Authoring - IPC Schema + `tables/` Refactor Plan + +## Summary + +Refactor dataset-authoring to use Arrow IPC **file format** for schemas and move build artifacts under `tables/`. Rename the authoring config field from `models` to `tables`. Keep Amp core unchanged by converting **legacy manifests → canonical package** in the dataset-authoring adapter layer when fetching from the admin API, and converting **package → legacy manifest JSON** in memory for `ampctl dataset register --package`. No backwards compatibility with old `sql/` or `*.schema.json` outputs. + +## Decisions + +- **Schema format**: Arrow IPC **file format** (`.ipc`), not JSON. +- **Build output layout**: + - `tables/.sql` for derived datasets only + - `tables/
.ipc` for all tables + - `functions/.js` unchanged +- **Authoring config**: rename `models` → `tables` (default `tables`). +- **Manifest table shape**: + - Derived table: `tables.
.sql` + `tables.
.ipc` + `network` + - Raw table: `tables.
.ipc` + `network` (no `sql` field) +- **Cache**: `~/.amp/registry//` stores canonical package format. +- **Interop**: + - **Admin API fetch**: legacy manifest JSON → canonical package (adapter). + - **Register**: package → legacy manifest JSON in memory (adapter). +- **No backwards compatibility** with old `sql/` + `*.schema.json` outputs. + +--- + +## Status + +| Phase | Description | Status | +|-------|-------------|--------| +| 1 | Arrow IPC Module | **Complete** | +| 2 | Rename `models` → `tables` | **Complete** | +| 3 | Build Output Layout (`sql/` → `tables/`) | **Complete** | +| 4 | Schema Type Refactor | **Complete** | +| 5 | Manifest Table Shape Changes | **Complete** | +| 6 | Adapter Layer (Legacy ↔ Package) | **Complete** | +| 7 | Cache Updates | **Complete** | +| 8 | Documentation & Tests | **Complete** | + +--- + +## Gap Analysis + +Based on codebase exploration (2026-02-04, verified via code search): + +### Currently Implemented + +| Component | Location | Details | +|-----------|----------|---------| +| Config parsing | `config.rs:87-118` | `AmpYaml` with `models: PathBuf` field, default via `default_models_dir()` returning `"models"` | +| Model discovery | `discovery.rs:97-162` | `discover_models()` scans `/**/*.sql`, returns `BTreeMap` | +| Build output | `manifest.rs:406,415` | `sql/
.sql` and `sql/
.schema.json` | +| Schema files | `arrow_json.rs` | JSON format using `ArrowSchema` from `datasets_common::manifest` | +| Package assembly | `package.rs:175-184` | Includes `sql/` and `functions/` directories | +| Cache | `cache.rs:131-133` | Stores `/manifest.json` only (no SQL/schema files) | +| Bridge | `bridge.rs` | Converts `AuthoringManifest` → legacy runtime format | +| Jinja | `jinja.rs` | `ref`, `source`, `var`, `env_var`, `this` template helpers | +| SQL validation | `query.rs` | SELECT-only, incremental mode constraints | +| Lockfile | `lockfile.rs` | `amp.lock` for reproducible dependency resolution | +| Validation | `validation.rs:105` | `discovered_models: BTreeMap` | +| TableDef | `manifest.rs:168-176` | Has `sql: FileRef`, `schema: FileRef`, `network: NetworkId` | +| Playground | `playground/` | Uses `models/` dir, builds to `build/sql/` | + +### Not Yet Implemented + +| Feature | Current State | Target State | +|---------|---------------|--------------| +| Arrow IPC I/O | `arrow_ipc.rs` module with `write_ipc_schema()`, `read_ipc_schema()` | **Complete** | +| Build output dir | `sql/` | `tables/` | +| Schema format | `.schema.json` (JSON) | `.ipc` (Arrow IPC file) | +| Config field | `models:` (default `"models"`) | `tables:` (default `"tables"`) | +| Raw table support | `sql` field required | `sql: Option` | +| Cache format | `manifest.json` only | Full package: `manifest.json` + `tables/` + `functions/` | +| Fetch adapter | N/A | Legacy manifest → canonical package conversion | +| Register adapter | N/A | Package → legacy manifest JSON for API | + +--- + +## Tasks + +### Phase 1: Arrow IPC Module (Foundational) + +**Files**: `arrow_ipc.rs` (new), `lib.rs` (1 line) + +**1.1) Create `arrow_ipc.rs` module** +- [x] Add new module `crates/core/dataset-authoring/src/arrow_ipc.rs` +- [x] Implement `write_ipc_schema(schema: &SchemaRef, path: &Path) -> Result<()>` + - Uses Arrow IPC FileWriter with schema-only (no record batches) +- [x] Implement `read_ipc_schema(path: &Path) -> Result` + - Uses Arrow IPC FileReader to read schema metadata +- [x] Add comprehensive tests for round-trip serialization +- [x] Export from `lib.rs` + +**Acceptance criteria**: Can write Arrow `Schema` to `.ipc` file and read it back losslessly. **VERIFIED** + +--- + +### Phase 2: Rename `models` → `tables` (Config Change) + +**Files**: `config.rs`, `discovery.rs`, `validation.rs`, CLI commands, integration tests + +**2.1) Update `config.rs`** +- [x] Rename field `models: PathBuf` to `tables: PathBuf` (line 109 in `AmpYaml`) +- [x] Update default function `default_models_dir()` → `default_tables_dir()` returning `"tables"` (line 156-158) +- [x] Update `AmpYamlV1` struct similarly (line 141) +- [x] Update `validate()` to reference "tables directory" instead of "models directory" (line 208) +- [x] Update all tests using `models` field (lines 507-852 test module) + +**2.2) Update `discovery.rs`** +- [x] Rename function `discover_models()` → `discover_tables()` (line 97) +- [x] Update variable name `models` → `tables` in function body (line 108) +- [x] Update `DiscoveryError` variants: `DuplicateModelName` → `DuplicateTableName`, etc. +- [x] Update `DiscoveredModel` → `DiscoveredTable` struct +- [x] Update documentation and error messages throughout + +**2.3) Update call sites** +- [x] Update `validation.rs` field `discovered_models` → `discovered_tables` (line 105 in `ValidationResult`) +- [x] Update `validate_network_inference()` parameters (line 591) +- [x] Update all callers of discovery functions in validation/build flows +- [x] Update CLI commands (check.rs, build.rs) to use new field/function names +- [x] Update integration tests (it_dataset_authoring.rs) with new paths and imports + +**Acceptance criteria**: `amp.yaml` accepts `tables:` field (with `tables` as default). `models:` is no longer recognized. **VERIFIED** + +--- + +### Phase 3: Build Output Layout (`sql/` → `tables/`) + +**Files**: `manifest.rs`, `package.rs`, `bridge.rs`, `arrow_json.rs`, CLI help text + +**3.1) Update `manifest.rs` output paths** +- [x] Change SQL file path from `sql/
.sql` to `tables/
.sql` (line 406) +- [x] Change schema file path from `sql/
.schema.json` to `tables/
.ipc` (line 415) +- [x] Update `sql_dir` parameter naming throughout to `tables_dir` +- [x] Update `ManifestBuilder` field `sql_dir: &'a Path` → `tables_dir: &'a Path` (line 286) +- [x] Update `ManifestBuilder::new()` parameter (line 306) +- [x] Update all test fixtures using `sql/` paths (lines 819-934 tests) + +**3.2) Update `package.rs`** +- [x] Change directory inclusion from `sql/` to `tables/` (lines 175-178) +- [x] Update `from_directory()` to look for `tables/` instead of `sql/` +- [x] Update all test fixtures and assertions (lines 613-651 tests) + +**3.3) Update `bridge.rs`** +- [x] Update all path references from `sql/` to `tables/` +- [x] Update test fixtures to use IPC schema files (completed in Phase 6) + +**3.4) Update `arrow_json.rs` → deprecate or remove** +- [x] After IPC is working, remove JSON schema write calls from build flow +- [x] Keep `arrow_json.rs` only if needed for legacy adapter layer in Phase 6 (confirmed needed) + +**Acceptance criteria**: `ampctl dataset build` produces `tables/
.sql` + `tables/
.ipc`, no `sql/` directory. **VERIFIED** + +--- + +### Phase 4: Schema Type Refactor (Arrow-native) + +**Files**: `schema.rs`, `validation.rs`, `dependency_manifest.rs`, build commands + +**4.1) Update schema inference in `schema.rs`** +- [x] Change return type from `TableSchema` to Arrow `SchemaRef` +- [x] Remove intermediate `TableSchema`/`ArrowSchema` conversions +- [x] Update `SchemaContext::infer_schema()` to return `SchemaRef` directly + +**4.2) Update build pipeline** +- [x] Write inferred schemas using `arrow_ipc::write_ipc_schema()` instead of JSON +- [x] Update validation output to use Arrow types (`ValidatedTable.schema: SchemaRef`) +- [x] Build command now uses `SchemaRef` directly without conversion + +**4.3) Update `dependency_manifest.rs`** +- [x] `DependencyTable.schema` stays as `TableSchema` for JSON serialization (needed for legacy adapter in Phase 6) +- [x] Added `From<&SchemaRef> for ArrowSchema` to enable conversion from native to serializable format + +**Acceptance criteria**: No `TableSchema`/`ArrowSchema` usage in authoring pipeline (only in adapter layer). **VERIFIED** + +--- + +### Phase 5: Manifest Table Shape Changes + +**Files**: `manifest.rs` (TableDef struct and builder) + +**5.1) Update `TableDef` in `manifest.rs`** +- [x] Change `schema: FileRef` to `ipc: FileRef` (line 173) +- [x] Make `sql: FileRef` optional: `sql: Option` (line 171) +- [x] Add `#[serde(skip_serializing_if = "Option::is_none")]` to `sql` field +- [x] Update `ManifestError::SchemaFileRef` → `ManifestError::IpcFileRef` (lines 78-87) + +**5.2) Update manifest builder** +- [x] Update `build_tables()` to create `TableDef` with optional `sql` (lines 401-438) +- [x] Handle derived tables: write both `sql: Some(...)` and `ipc: ...` +- [x] Prepare for raw tables: `sql: None`, only `ipc` field (future support) + +**5.3) Update tests** +- [x] Update `table_def_serializes_correctly` test (lines 727-753) +- [x] Add test for optional SQL field serialization +- [x] Add test for raw table (no SQL) serialization + +**Acceptance criteria**: Manifest JSON has `"ipc"` field instead of `"schema"`. SQL field can be optional. + +--- + +### Phase 6: Adapter Layer - Legacy ↔ Package + +**Files**: `bridge.rs` (extend), `cache.rs` (adapter calls), `resolver.rs` (use adapter) + +**6.1) Admin API fetch adapter (legacy → package)** +- [x] Create adapter function to convert legacy manifest JSON to canonical package format +- [x] When fetching from admin API: + - Parse legacy `manifest.json` + - Extract inline SQL content → write to `tables/
.sql` + - Convert inline schema JSON → write to `tables/
.ipc` + - Copy function sources → `functions/` + - Write canonical `manifest.json` with file refs +- [x] Store canonical package in cache directory (via `LegacyAdapter` writing to target dir) + +**6.2) Register adapter (package → legacy)** +- [x] Create adapter function to convert package format to legacy manifest JSON +- [x] When registering via `--package`: + - Read `tables/
.ipc` → convert to legacy schema JSON + - Read `tables/
.sql` content + - Read `functions/` sources + - Build legacy manifest JSON with inline content +- [x] Upload legacy manifest to admin API (existing functionality via `LegacyBridge::to_json()`) + +**6.3) Constrain legacy parsing** +- [x] Ensure all legacy JSON schema parsing is confined to adapter layer +- [x] Update resolver to use LegacyAdapter for derived datasets (kind="manifest") +- [x] Raw datasets (evm-rpc, firehose, etc.) parsed directly into DependencyManifest + +**Acceptance criteria**: +- Fetching legacy manifests populates cache with canonical package format. +- Registering a package produces valid legacy manifest JSON for the API. + +--- + +### Phase 7: Cache Updates + +**Files**: `cache.rs`, `resolver.rs`, `dependency_manifest.rs` + +**7.1) Update `cache.rs`** +- [x] Cache structure stores full package format (via LegacyAdapter): + - `/manifest.json` (DependencyManifest) + - `/tables/
.sql` + - `/tables/
.ipc` + - `/functions/.js` +- [x] Add `CachedPackage` struct with methods: + - `manifest()` - returns `&DependencyManifest` + - `read_sql(table_name)` - reads SQL content + - `read_schema(table_name)` - reads Arrow SchemaRef from IPC file + - `read_function(filename)` - reads function source + - `has_sql()`, `has_ipc_schema()` - existence checks +- [x] Add `Cache::get_package()` returning `Option` +- [x] Keep existing `Cache::get()` for backward compatibility (returns `DependencyManifest`) +- [x] Add error variants for IPC/SQL/function file reads + +**7.2) Update `resolver.rs`** (No changes needed) +- [x] Resolver already works correctly: + - Uses `LegacyAdapter` to write full package to cache directory + - Uses `Cache::put()` to store `DependencyManifest` + - Uses `Cache::get()` to retrieve `DependencyManifest` for resolution + - Consumers can use `Cache::get_package()` when IPC access is needed + +**7.3) Update `dependency_manifest.rs`** (Deferred) +- [x] `DependencyTable.schema` keeps `TableSchema` (JSON-serializable) for compatibility +- [x] Consumers needing Arrow SchemaRef use `CachedPackage::read_schema()` +- Note: Lazy loading from IPC considered but not needed - current approach works + +**Acceptance criteria**: Cache stores and retrieves full canonical packages, not just manifest JSON. **VERIFIED** + +--- + +### Phase 8: Documentation & Tests + +**Files**: `docs/features/dataset-authoring.md`, `tests/src/tests/it_dataset_authoring.rs`, CLI help markdown files, `playground/` + +**8.1) Update `docs/features/dataset-authoring.md`** +- [x] Replace all `models/` references with `tables/` (currently 11 occurrences) +- [x] Update build output structure section (currently shows `sql/` layout) +- [x] Update `amp.yaml` schema documentation (config field `models` → `tables`) +- [x] Document `.ipc` schema format (replace `.schema.json` references) +- [x] Update CLI examples + +**8.2) Update tests** +- [x] Update all fixture paths from `sql/` to `tables/` +- [x] Update all `.schema.json` references to `.ipc` +- [x] Add IPC round-trip tests (already exist in arrow_ipc.rs) +- [x] Add adapter layer tests for legacy conversion (already exist in bridge.rs) +- [x] Ensure coverage of new table shape (optional sql) (already exist in manifest.rs) + +**8.3) Update CLI help text** +- [x] Update `ampctl dataset` subcommand help for `tables/` directory (package.rs docstrings updated) + +**8.4) Update playground sample** +- [x] Delete `playground/build/` directory (gitignored, not tracked) +- [x] Rename `playground/models/` to `playground/tables/` +- [x] Update `playground/amp.yaml` to use `tables:` field (relies on new default) +- [x] Regenerate `playground/build/` with new structure (gitignored) + +**8.5) Update module docstrings** +- [x] Update `lib.rs` docstring (lines 1-29) mentioning `models/` and `sql/` +- [x] Update `manifest.rs` docstring (lines 1-41) with example JSON + +**Acceptance criteria**: Docs, tests, and samples are consistent with new implementation. **VERIFIED** + +--- + +## File Annotations + +Quick reference for key files and line numbers (verified 2026-02-04): + +| File | Key Lines | Purpose | +|------|-----------|---------| +| `config.rs` | 87-118, 156-158 | `AmpYaml` struct, `default_models_dir()` | +| `discovery.rs` | 97-162 | `discover_models()` function | +| `manifest.rs` | 168-176, 282-292 | `TableDef` struct, `ManifestBuilder` | +| `package.rs` | 164-187 | `from_directory()` reads `sql/` and `functions/` | +| `cache.rs` | 131-133, 145-163, 173-200 | `manifest_path()`, `get()`, `put()` | +| `arrow_json.rs` | 74-104 | `write_schema_file()`, `read_schema_file()` | +| `validation.rs` | 105, 429-470, 591 | `discovered_models` field and usage | +| `lib.rs` | 1-29 | Module docstring with workflow description | +| `files.rs` | docstrings, tests | Path examples use `sql/` throughout | +| `bridge.rs` | tests | Test fixtures use `sql/` paths extensively | +| `playground/amp.yaml` | 1-8 | Sample config using `models` default | + +--- + +## Implementation Order + +**Recommended sequence** (minimizes rework): + +1. **Phase 1** - Arrow IPC module (no dependencies, foundational) +2. **Phase 5.1** - TableDef field rename (`schema` → `ipc`, optional `sql`) +3. **Phase 3** - Build output layout change (`sql/` → `tables/`) +4. **Phase 2** - Config rename (`models` → `tables`) +5. **Phase 4** - Schema type refactor (use Arrow-native types) +6. **Phase 6** - Adapter layer (legacy conversion) +7. **Phase 7** - Cache format update +8. **Phase 8** - Documentation and tests + +Each phase should be completable in one commit/PR. + +--- + +## Answered Questions + +1. **Deprecation period for `models`?** - No backwards compatibility. No deprecation period. None of this code is released yet. +2. **Cache migration?** - No migration. Clear old cache entries. No backwards compatibility. None of this code is released yet. +3. **Raw table authoring?** - Raw datasets are currently defined by extractor code. Eventually this will change and raw datasets will also have their table schemas declared and discoverable in a registry. But this is out of scope for this plan. + +--- + +## Blockers + +None identified. All dependencies (Arrow, IPC support) are already available in the workspace. +- Arrow IPC verified in use at: `crates/services/server/src/flight.rs:595`, `crates/clients/flight/src/store/mod.rs:195` + +--- + +## Next Steps + +All phases complete. The refactor is done: +- Arrow IPC schema format in use +- `tables/` directory layout +- `tables:` config field (with `tables` default) +- Legacy adapter layer for API interop +- Full package caching + +Ready for review and merge. diff --git a/crates/bin/ampctl/Cargo.toml b/crates/bin/ampctl/Cargo.toml index 10c1c4668..979f593b1 100644 --- a/crates/bin/ampctl/Cargo.toml +++ b/crates/bin/ampctl/Cargo.toml @@ -17,8 +17,12 @@ path = "src/main.rs" admin-client = { path = "../../clients/admin" } amp-object-store = { path = "../../core/object-store" } anyhow.workspace = true +async-trait.workspace = true clap.workspace = true +common = { path = "../../core/common" } console = "0.16.1" +datafusion.workspace = true +dataset-authoring = { path = "../../core/dataset-authoring" } datasets-common = { path = "../../core/datasets-common" } datasets-derived = { path = "../../core/datasets-derived" } dump = { path = "../../core/dump" } diff --git a/crates/bin/ampctl/src/cmd/dataset.rs b/crates/bin/ampctl/src/cmd/dataset.rs index 320de3084..159b3c5f8 100644 --- a/crates/bin/ampctl/src/cmd/dataset.rs +++ b/crates/bin/ampctl/src/cmd/dataset.rs @@ -1,9 +1,13 @@ //! Dataset management commands +mod authoring; +pub mod build; +pub mod check; pub mod deploy; pub mod inspect; pub mod list; pub mod manifest; +pub mod package; pub mod register; pub mod restore; pub mod versions; @@ -11,6 +15,28 @@ pub mod versions; /// Dataset management subcommands. #[derive(Debug, clap::Subcommand)] pub enum Commands { + /// Build a dataset from amp.yaml configuration + /// + /// Parses the amp.yaml configuration, resolves dependencies, renders + /// Jinja SQL templates, validates CTAS statements, infers schemas, + /// and writes output files to the target directory. + Build(build::Args), + + /// Validate dataset authoring inputs without building artifacts + /// + /// Parses the amp.yaml/amp.yml configuration, resolves dependencies, + /// renders Jinja SQL templates, validates SELECT statements, and + /// infers schemas without writing output files. + #[command(after_help = include_str!("dataset/check__after_help.md"))] + Check(check::Args), + + /// Package built dataset artifacts into a .tgz archive + /// + /// Creates a deterministic archive containing manifest.json, SQL files, + /// schemas, and function sources for distribution or deployment. + #[command(alias = "pkg")] + Package(package::Args), + /// Deploy a dataset to start syncing blockchain data /// /// Deploys a dataset version by scheduling a data extraction job via the @@ -59,6 +85,9 @@ pub enum Commands { /// Execute the dataset command with the given subcommand. pub async fn run(command: Commands) -> anyhow::Result<()> { match command { + Commands::Build(args) => build::run(args).await?, + Commands::Check(args) => check::run(args).await?, + Commands::Package(args) => package::run(args).await?, Commands::Deploy(args) => deploy::run(args).await?, Commands::Register(args) => register::run(args).await?, Commands::List(args) => list::run(args).await?, diff --git a/crates/bin/ampctl/src/cmd/dataset/authoring.rs b/crates/bin/ampctl/src/cmd/dataset/authoring.rs new file mode 100644 index 000000000..c68baabc7 --- /dev/null +++ b/crates/bin/ampctl/src/cmd/dataset/authoring.rs @@ -0,0 +1,140 @@ +//! Shared helpers for dataset authoring commands. + +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, + sync::Arc, +}; + +use async_trait::async_trait; +use dataset_authoring::{ + lockfile::{Lockfile, LockfileError, RootInfo}, + resolver::{ManifestFetcher, ResolveError}, +}; +use datasets_common::{hash::Hash, reference::Reference}; +use datasets_derived::deps::{DepAlias, DepReference, HashOrVersion}; + +use crate::args::GlobalArgs; + +/// Parse a NAME=VALUE string into a tuple. +pub(super) fn parse_var(s: &str) -> Result<(String, String), String> { + let pos = s + .find('=') + .ok_or_else(|| format!("invalid variable format: expected NAME=VALUE, got '{s}'"))?; + Ok((s[..pos].to_string(), s[pos + 1..].to_string())) +} + +pub(super) fn dependencies_have_version_refs( + dependencies: &BTreeMap, +) -> bool { + dependencies + .values() + .any(|reference| matches!(reference.revision(), HashOrVersion::Version(_))) +} + +pub(super) fn load_lockfile_for_locked( + lockfile_path: &Path, + config: &dataset_authoring::config::AmpYaml, +) -> Result { + if !lockfile_path.exists() { + return Err(LockfileLoadError::LockfileRequired); + } + + let lockfile = Lockfile::read(lockfile_path).map_err(LockfileLoadError::Lockfile)?; + let root = RootInfo { + namespace: config.namespace.clone(), + name: config.name.clone(), + version: config.version.clone(), + }; + lockfile + .verify_root(&root) + .map_err(LockfileLoadError::Lockfile)?; + lockfile + .verify_dependencies(&config.dependencies) + .map_err(LockfileLoadError::Lockfile)?; + + Ok(lockfile) +} + +pub(super) fn load_lockfile_for_offline( + lockfile_path: &Path, +) -> Result { + if !lockfile_path.exists() { + return Err(LockfileLoadError::OfflineLockfileRequired( + lockfile_path.to_path_buf(), + )); + } + + Lockfile::read(lockfile_path).map_err(LockfileLoadError::Lockfile) +} + +/// Admin API manifest fetcher. +pub(super) struct AdminApiFetcher { + client: Arc, +} + +impl AdminApiFetcher { + pub(super) fn new(global: &GlobalArgs) -> Result { + let client = global.build_client()?; + Ok(Self { + client: Arc::new(client), + }) + } +} + +#[async_trait] +impl ManifestFetcher for AdminApiFetcher { + async fn fetch_by_version( + &self, + reference: &DepReference, + ) -> Result { + // Convert DepReference to Reference for the datasets client + let ref_str = reference.to_string(); + let datasets_ref: Reference = ref_str.parse().map_err(|e| ResolveError::FetchError { + reference: ref_str.clone(), + message: format!("invalid reference format: {e}"), + })?; + + // Use the datasets client to fetch manifest by reference + let result = self + .client + .datasets() + .get_manifest(&datasets_ref) + .await + .map_err(|e| ResolveError::FetchError { + reference: ref_str.clone(), + message: e.to_string(), + })?; + + result.ok_or(ResolveError::NotFound { reference: ref_str }) + } + + async fn fetch_by_hash(&self, hash: &Hash) -> Result, ResolveError> { + self.client + .manifests() + .get(hash) + .await + .map_err(|e| ResolveError::FetchError { + reference: hash.to_string(), + message: e.to_string(), + }) + } +} + +/// Errors for loading/validating amp.lock in authoring commands. +#[derive(Debug, thiserror::Error)] +pub(super) enum LockfileLoadError { + /// Lockfile required but not found (--locked mode). + #[error("amp.lock not found (required for --locked mode)")] + LockfileRequired, + + /// Lockfile required for offline version resolution. + #[error( + "offline mode requires amp.lock when version refs are present; add amp.lock, pin dependencies by hash, or run without --offline (expected at {0})" + )] + OfflineLockfileRequired(PathBuf), + + /// Lockfile operation failed. + #[error("lockfile error")] + Lockfile(#[source] LockfileError), +} diff --git a/crates/bin/ampctl/src/cmd/dataset/build.rs b/crates/bin/ampctl/src/cmd/dataset/build.rs new file mode 100644 index 000000000..84d54aec0 --- /dev/null +++ b/crates/bin/ampctl/src/cmd/dataset/build.rs @@ -0,0 +1,395 @@ +//! Dataset build command. +//! +//! Builds a derived dataset from `amp.yaml` configuration by: +//! 1. Parsing the `amp.yaml` configuration file +//! 2. Discovering tables from the `tables/` directory +//! 3. Resolving dependencies from the admin API +//! 4. Rendering Jinja SQL templates with context +//! 5. Validating SELECT statements and incremental constraints +//! 6. Inferring table schemas via DataFusion planning +//! 7. Writing output files to the output directory: +//! - `manifest.json` (canonical) +//! - `tables/
.sql` (rendered SQL) +//! - `tables/
.ipc` (inferred schema) +//! - `functions/` (if any) +//! 8. Updating or creating `amp.lock` lockfile +//! +//! # Configuration +//! +//! - Admin URL: `--admin-url` flag or `AMP_ADMIN_URL` env var (default: `http://localhost:1610`) +//! - Logging: `AMP_LOG` env var (`error`, `warn`, `info`, `debug`, `trace`) + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use dataset_authoring::{ + arrow_ipc, + cache::Cache, + config::{self, AmpYaml}, + lockfile::{Lockfile, LockfileError, RootInfo}, + manifest::ManifestBuilder, + resolver::{DependencyGraph, Resolver}, + validation::{self, ResolutionMode, ValidationOptions}, +}; +use datasets_common::table_name::TableName; + +use super::authoring; +use crate::args::GlobalArgs; + +/// Command-line arguments for the `dataset build` command. +#[derive(Debug, clap::Args)] +#[command( + about = "Build a dataset from amp.yaml configuration", + after_help = include_str!("build__after_help.md") +)] +pub struct Args { + #[command(flatten)] + pub global: GlobalArgs, + + /// Directory containing amp.yaml (defaults to current directory). + #[arg(long, short = 'd', default_value = ".")] + pub dir: PathBuf, + + /// Output directory for build artifacts (required). + #[arg(long, short = 'o', required = true)] + pub output: PathBuf, + + /// Variable overrides in the form NAME=VALUE. + /// + /// These take precedence over variables defined in amp.yaml. + #[arg(long = "var", short = 'V', value_parser = authoring::parse_var)] + pub vars: Vec<(String, String)>, + + /// Fail if amp.lock is missing or doesn't match resolved dependencies. + #[arg(long)] + pub locked: bool, + + /// Disable network fetches; requires amp.lock when version refs are present; cache misses are errors. + #[arg(long)] + pub offline: bool, + + /// Equivalent to --locked + --offline. + #[arg(long)] + pub frozen: bool, +} + +/// Result of a dataset build operation. +#[derive(serde::Serialize)] +struct BuildResult { + status: &'static str, + output: String, + manifest_hash: String, + tables: usize, + functions: usize, +} + +impl std::fmt::Display for BuildResult { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "Build complete")?; + writeln!(f, " Output: {}", self.output)?; + writeln!(f, " Manifest hash: {}", self.manifest_hash)?; + writeln!(f, " Tables: {}", self.tables)?; + if self.functions > 0 { + writeln!(f, " Functions: {}", self.functions)?; + } + Ok(()) + } +} + +/// Build a dataset from amp.yaml configuration. +/// +/// # Errors +/// +/// Returns [`Error`] for configuration errors, resolution failures, +/// SQL validation errors, or I/O failures. +#[tracing::instrument(skip_all, fields(dir = %args.dir.display()))] +pub async fn run(args: Args) -> Result<(), Error> { + let Args { + global, + dir, + output, + vars, + locked, + offline, + frozen, + } = args; + let locked = locked || frozen; + let offline = offline || frozen; + + // Ensure dir exists and is a directory + if !dir.is_dir() { + return Err(Error::InvalidDirectory { path: dir }); + } + + // 1. Parse amp.yaml/amp.yml (needed for lockfile mode selection) + tracing::info!("Parsing authoring config"); + let (config_path, config) = config::load_from_dir(&dir)?; + tracing::debug!(path = %config_path.display(), "Resolved config path"); + + let lockfile_path = Lockfile::path_in(&dir); + + // 2. Setup resolver with admin API fetcher + let cache = Cache::new().map_err(Error::CacheSetup)?; + let fetcher = authoring::AdminApiFetcher::new(&global).map_err(Error::ClientBuildError)?; + let resolver = Resolver::new(fetcher, cache).with_offline(offline); + + // 3. Determine resolution mode + let lockfile = if locked { + Some(authoring::load_lockfile_for_locked( + &lockfile_path, + &config, + )?) + } else if offline && authoring::dependencies_have_version_refs(&config.dependencies) { + Some(authoring::load_lockfile_for_offline(&lockfile_path)?) + } else { + None + }; + + let resolution = match lockfile.as_ref() { + Some(lockfile) => ResolutionMode::Locked { lockfile }, + None => ResolutionMode::Resolve, + }; + + // 4. Run shared validation pipeline + tracing::info!("Validating dataset project"); + let validation_output = validation::validate_project_with_config( + ValidationOptions::new(&dir, &resolver) + .with_cli_vars(&vars) + .with_resolution(resolution), + config_path, + config, + ) + .await + .map_err(Error::Validation)?; + + let validation::ValidationOutput { + config, + discovered_tables, + dependencies: dep_graph, + validated_tables, + .. + } = validation_output; + + tracing::debug!( + namespace = %config.namespace, + name = %config.name, + version = %config.version, + tables = discovered_tables.len(), + "Loaded configuration" + ); + tracing::debug!( + direct_deps = dep_graph.direct.len(), + transitive_deps = dep_graph.len(), + "Resolved dependencies" + ); + + // 5. Handle lockfile + if !locked { + handle_lockfile(&lockfile_path, &config, &dep_graph)?; + } + + // 6. Create output directories + let tables_dir = output.join("tables"); + let functions_dir = output.join("functions"); + fs::create_dir_all(&tables_dir).map_err(|e| Error::CreateDirectory { + path: tables_dir.clone(), + source: e, + })?; + + // 7. Write table artifacts + tracing::info!("Writing table artifacts"); + for (table_name, table) in &validated_tables { + let sql_output_path = tables_dir.join(format!("{table_name}.sql")); + fs::write(&sql_output_path, &table.rendered_sql).map_err(|e| Error::WriteFile { + path: sql_output_path, + source: e, + })?; + + let ipc_output_path = tables_dir.join(format!("{table_name}.ipc")); + arrow_ipc::write_ipc_schema(&table.schema, &ipc_output_path).map_err(|e| { + Error::WriteIpcSchema { + table: table_name.clone(), + source: e, + } + })?; + } + + // 8. Copy function files if present + if let Some(functions) = &config.functions { + fs::create_dir_all(&functions_dir).map_err(|e| Error::CreateDirectory { + path: functions_dir.clone(), + source: e, + })?; + + for (func_name, func_def) in functions { + let source_path = dir.join(&func_def.source); + let dest_path = functions_dir.join(format!("{}.js", func_name)); + fs::copy(&source_path, &dest_path).map_err(|e| Error::CopyFile { + source: source_path, + dest: dest_path, + error: e, + })?; + } + } + + // 9. Generate manifest + tracing::info!("Generating manifest"); + let manifest = ManifestBuilder::new( + &config, + &dep_graph, + &output, + &tables_dir, + &discovered_tables, + ) + .build() + .map_err(Error::ManifestGeneration)?; + + // Write manifest.json (canonical JSON for identity verification) + let manifest_json = manifest + .canonical_json() + .map_err(Error::ManifestGeneration)?; + let manifest_path = output.join("manifest.json"); + fs::write(&manifest_path, &manifest_json).map_err(|e| Error::WriteFile { + path: manifest_path, + source: e, + })?; + + let identity = manifest.identity().map_err(Error::ManifestGeneration)?; + tracing::info!(hash = %identity, "Build complete"); + + let functions = config + .functions + .as_ref() + .map(|funcs| funcs.len()) + .unwrap_or(0); + let result = BuildResult { + status: "ok", + output: output.display().to_string(), + manifest_hash: identity.to_string(), + tables: discovered_tables.len(), + functions, + }; + global.print(&result).map_err(Error::JsonFormattingError)?; + + Ok(()) +} + +fn handle_lockfile( + lockfile_path: &Path, + config: &AmpYaml, + dep_graph: &DependencyGraph, +) -> Result<(), Error> { + let root = RootInfo { + namespace: config.namespace.clone(), + name: config.name.clone(), + version: config.version.clone(), + }; + + // Update lockfile if deps have changed + let new_lockfile = Lockfile::from_graph(root, dep_graph); + let should_write = if lockfile_path.exists() { + let existing = Lockfile::read(lockfile_path).map_err(Error::Lockfile)?; + existing != new_lockfile + } else { + true + }; + if should_write { + new_lockfile.write(lockfile_path).map_err(Error::Lockfile)?; + tracing::info!("Updated amp.lock"); + } + + Ok(()) +} + +/// Errors for dataset build operations. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Invalid directory specified. + #[error("not a directory: {}", path.display())] + InvalidDirectory { path: PathBuf }, + + /// Failed to locate, read, or parse the authoring config. + #[error(transparent)] + ConfigLoad(#[from] dataset_authoring::config::ConfigLoadError), + + /// Failed to setup cache. + #[error("failed to setup cache")] + CacheSetup(#[source] dataset_authoring::cache::CacheError), + + /// Failed to build admin API client. + #[error("failed to build admin API client")] + ClientBuildError(#[source] crate::args::BuildClientError), + + /// Dataset validation failed. + #[error(transparent)] + Validation(#[from] dataset_authoring::validation::ValidationError), + + /// Lockfile required but not found (--locked mode). + #[error("amp.lock not found (required for --locked mode)")] + LockfileRequired, + + /// Lockfile required for offline version resolution. + #[error( + "offline mode requires amp.lock when version refs are present; add amp.lock, pin dependencies by hash, or run without --offline (expected at {0})" + )] + OfflineLockfileRequired(PathBuf), + + /// Lockfile operation failed. + #[error("lockfile error")] + Lockfile(#[source] LockfileError), + + /// Failed to create directory. + #[error("failed to create directory {}", path.display())] + CreateDirectory { + path: PathBuf, + #[source] + source: std::io::Error, + }, + + /// Failed to write file. + #[error("failed to write file {}", path.display())] + WriteFile { + path: PathBuf, + #[source] + source: std::io::Error, + }, + + /// Failed to write IPC schema file. + #[error("failed to write IPC schema for table '{table}'")] + WriteIpcSchema { + table: TableName, + #[source] + source: dataset_authoring::arrow_ipc::IpcSchemaError, + }, + + /// Failed to copy function file. + #[error("failed to copy function file from {} to {}", source.display(), dest.display())] + CopyFile { + source: PathBuf, + dest: PathBuf, + #[source] + error: std::io::Error, + }, + + /// Manifest generation failed. + #[error("manifest generation failed")] + ManifestGeneration(#[source] dataset_authoring::manifest::ManifestError), + + /// Failed to format build JSON. + #[error("failed to format build JSON")] + JsonFormattingError(#[source] serde_json::Error), +} + +impl From for Error { + fn from(err: authoring::LockfileLoadError) -> Self { + match err { + authoring::LockfileLoadError::LockfileRequired => Error::LockfileRequired, + authoring::LockfileLoadError::OfflineLockfileRequired(path) => { + Error::OfflineLockfileRequired(path) + } + authoring::LockfileLoadError::Lockfile(source) => Error::Lockfile(source), + } + } +} diff --git a/crates/bin/ampctl/src/cmd/dataset/build__after_help.md b/crates/bin/ampctl/src/cmd/dataset/build__after_help.md new file mode 100644 index 000000000..236e90293 --- /dev/null +++ b/crates/bin/ampctl/src/cmd/dataset/build__after_help.md @@ -0,0 +1,73 @@ +## Examples + +Build dataset in current directory: +``` +ampctl dataset build --output build/ +``` + +Build dataset from specific directory: +``` +ampctl dataset build --dir my_dataset/ --output build/ +``` + +Build with custom output directory: +``` +ampctl dataset build --dir my_dataset/ --output dist/ +``` + +Override template variables: +``` +ampctl dataset build --output build/ --var network=mainnet --var chain_id=1 +``` + +Build with locked dependencies (fail if amp.lock missing or stale): +``` +ampctl dataset build --output build/ --locked +``` + +Build with offline cache only (requires amp.lock when version refs are present): +``` +ampctl dataset build --output build/ --offline +``` + +Build with frozen dependencies (equivalent to --locked + --offline): +``` +ampctl dataset build --output build/ --frozen +``` + +Build with custom admin URL: +``` +ampctl dataset build --output build/ --admin-url http://production:1610 +``` + +## Build Workflow + +1. Parses `amp.yaml` or `amp.yml` configuration file +2. Resolves dependencies from the admin API +3. Renders Jinja SQL templates with variables +4. Validates SELECT-only statements and incremental constraints +5. Infers table schemas via DataFusion planning +6. Writes output files to the target directory + +## Output Structure + +``` +/ + manifest.json # Canonical manifest with file refs + sql/ +
.sql # Rendered SQL (SELECT statement) +
.schema.json # Inferred Arrow schema + functions/ + .js # Copied function sources +``` + +## Lockfile Behavior + +- **Default**: Creates/updates `amp.lock` if dependencies changed +- **--locked**: Fails if lockfile missing or dependencies don't match +- **--offline**: Disables network fetches; requires `amp.lock` when version refs are present; cache misses are errors +- **--frozen**: Equivalent to `--locked` + `--offline` + +## Structured Output + +Use `--json` to emit machine-readable output for automation. diff --git a/crates/bin/ampctl/src/cmd/dataset/check.rs b/crates/bin/ampctl/src/cmd/dataset/check.rs new file mode 100644 index 000000000..d86d37f51 --- /dev/null +++ b/crates/bin/ampctl/src/cmd/dataset/check.rs @@ -0,0 +1,257 @@ +//! Dataset check command. +//! +//! Validates a derived dataset project without writing build artifacts by: +//! 1. Parsing the authoring configuration +//! 2. Discovering tables from the `tables/` directory +//! 3. Resolving dependencies from the admin API +//! 4. Rendering Jinja SQL templates with context +//! 5. Validating SELECT statements and incremental constraints +//! 6. Inferring table schemas via DataFusion planning + +use std::path::PathBuf; + +use dataset_authoring::{ + cache::Cache, + config, + lockfile::{Lockfile, LockfileError}, + resolver::Resolver, + validation::{self, ResolutionMode, ValidationOptions}, +}; + +use super::authoring; +use crate::args::GlobalArgs; + +/// Command-line arguments for the `dataset check` command. +#[derive(Debug, clap::Args)] +#[command( + about = "Validate dataset authoring inputs without building artifacts", + after_help = include_str!("check__after_help.md") +)] +pub struct Args { + #[command(flatten)] + pub global: GlobalArgs, + + /// Directory containing amp.yaml/amp.yml (defaults to current directory). + #[arg(long, short = 'd', default_value = ".")] + pub dir: PathBuf, + + /// Variable overrides in the form NAME=VALUE. + /// + /// These take precedence over variables defined in amp.yaml. + #[arg(long = "var", short = 'V', value_parser = authoring::parse_var)] + pub vars: Vec<(String, String)>, + + /// Fail if amp.lock is missing or doesn't match resolved dependencies. + #[arg(long)] + pub locked: bool, + + /// Disable network fetches; requires amp.lock when version refs are present; cache misses are errors. + #[arg(long)] + pub offline: bool, + + /// Equivalent to --locked + --offline. + #[arg(long)] + pub frozen: bool, +} + +/// Result of a dataset check operation. +#[derive(serde::Serialize)] +struct CheckResult { + status: &'static str, + config_path: String, + namespace: String, + name: String, + version: String, + tables: usize, + dependencies: DependencySummary, + functions: usize, +} + +#[derive(serde::Serialize)] +struct DependencySummary { + direct: usize, + transitive: usize, +} + +impl std::fmt::Display for CheckResult { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "Check complete")?; + writeln!(f, " Config: {}", self.config_path)?; + writeln!( + f, + " Dataset: {}/{} v{}", + self.namespace, self.name, self.version + )?; + writeln!(f, " Tables: {}", self.tables)?; + writeln!( + f, + " Dependencies: {} direct, {} total", + self.dependencies.direct, self.dependencies.transitive + )?; + if self.functions > 0 { + writeln!(f, " Functions: {}", self.functions)?; + } + Ok(()) + } +} + +/// Validate a dataset authoring project without writing build artifacts. +/// +/// # Errors +/// +/// Returns [`Error`] for configuration errors, resolution failures, +/// SQL validation errors, or I/O failures. +#[tracing::instrument(skip_all, fields(dir = %args.dir.display()))] +pub async fn run(args: Args) -> Result<(), Error> { + let Args { + global, + dir, + vars, + locked, + offline, + frozen, + } = args; + let locked = locked || frozen; + let offline = offline || frozen; + + // Ensure dir exists and is a directory + if !dir.is_dir() { + return Err(Error::InvalidDirectory { path: dir }); + } + + // 1. Parse amp.yaml/amp.yml (needed for lockfile mode selection) + tracing::info!("Parsing authoring config"); + let (config_path, config) = config::load_from_dir(&dir)?; + tracing::debug!(path = %config_path.display(), "Resolved config path"); + + let lockfile_path = Lockfile::path_in(&dir); + + // 2. Setup resolver with admin API fetcher + let cache = Cache::new().map_err(Error::CacheSetup)?; + let fetcher = authoring::AdminApiFetcher::new(&global).map_err(Error::ClientBuildError)?; + let resolver = Resolver::new(fetcher, cache).with_offline(offline); + + // 3. Determine resolution mode + let lockfile = if locked { + Some(authoring::load_lockfile_for_locked( + &lockfile_path, + &config, + )?) + } else if offline && authoring::dependencies_have_version_refs(&config.dependencies) { + Some(authoring::load_lockfile_for_offline(&lockfile_path)?) + } else { + None + }; + + let resolution = match lockfile.as_ref() { + Some(lockfile) => ResolutionMode::Locked { lockfile }, + None => ResolutionMode::Resolve, + }; + + // 4. Run shared validation pipeline + tracing::info!("Validating dataset project"); + let validation_output = validation::validate_project_with_config( + ValidationOptions::new(&dir, &resolver) + .with_cli_vars(&vars) + .with_resolution(resolution), + config_path, + config, + ) + .await + .map_err(Error::Validation)?; + + let validation::ValidationOutput { + config_path, + config, + discovered_tables, + dependencies: dep_graph, + .. + } = validation_output; + + tracing::debug!( + namespace = %config.namespace, + name = %config.name, + version = %config.version, + tables = discovered_tables.len(), + "Loaded configuration" + ); + tracing::debug!( + direct_deps = dep_graph.direct.len(), + transitive_deps = dep_graph.len(), + "Resolved dependencies" + ); + + let functions = config.functions.as_ref().map(|f| f.len()).unwrap_or(0); + + let result = CheckResult { + status: "ok", + config_path: config_path.display().to_string(), + namespace: config.namespace.to_string(), + name: config.name.to_string(), + version: config.version.to_string(), + tables: discovered_tables.len(), + dependencies: DependencySummary { + direct: dep_graph.direct.len(), + transitive: dep_graph.len(), + }, + functions, + }; + + global.print(&result).map_err(Error::JsonFormattingError)?; + + Ok(()) +} + +/// Errors for dataset check operations. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Invalid directory specified. + #[error("not a directory: {}", path.display())] + InvalidDirectory { path: PathBuf }, + + /// Failed to locate, read, or parse the authoring config. + #[error(transparent)] + ConfigLoad(#[from] dataset_authoring::config::ConfigLoadError), + + /// Failed to setup cache. + #[error("failed to setup cache")] + CacheSetup(#[source] dataset_authoring::cache::CacheError), + + /// Failed to build admin API client. + #[error("failed to build admin API client")] + ClientBuildError(#[source] crate::args::BuildClientError), + + /// Dataset validation failed. + #[error(transparent)] + Validation(#[from] dataset_authoring::validation::ValidationError), + + /// Lockfile required but not found (--locked mode). + #[error("amp.lock not found (required for --locked mode)")] + LockfileRequired, + + /// Lockfile required for offline version resolution. + #[error( + "offline mode requires amp.lock when version refs are present; add amp.lock, pin dependencies by hash, or run without --offline (expected at {0})" + )] + OfflineLockfileRequired(PathBuf), + + /// Lockfile operation failed. + #[error("lockfile error")] + Lockfile(#[source] LockfileError), + + /// Failed to format JSON for display. + #[error("failed to format check JSON")] + JsonFormattingError(#[source] serde_json::Error), +} + +impl From for Error { + fn from(err: authoring::LockfileLoadError) -> Self { + match err { + authoring::LockfileLoadError::LockfileRequired => Error::LockfileRequired, + authoring::LockfileLoadError::OfflineLockfileRequired(path) => { + Error::OfflineLockfileRequired(path) + } + authoring::LockfileLoadError::Lockfile(source) => Error::Lockfile(source), + } + } +} diff --git a/crates/bin/ampctl/src/cmd/dataset/check__after_help.md b/crates/bin/ampctl/src/cmd/dataset/check__after_help.md new file mode 100644 index 000000000..8b0e73c5d --- /dev/null +++ b/crates/bin/ampctl/src/cmd/dataset/check__after_help.md @@ -0,0 +1,48 @@ +## Examples + +Validate dataset in current directory: +``` +ampctl dataset check +``` + +Validate dataset from specific directory: +``` +ampctl dataset check --dir my_dataset/ +``` + +Override template variables: +``` +ampctl dataset check --var network=mainnet --var chain_id=1 +``` + +Validate with locked dependencies (fail if amp.lock missing or stale): +``` +ampctl dataset check --locked +``` + +Validate with offline cache only (requires amp.lock when version refs are present): +``` +ampctl dataset check --offline +``` + +Validate with frozen dependencies (equivalent to --locked + --offline): +``` +ampctl dataset check --frozen +``` + +Machine-readable output: +``` +ampctl dataset check --json +``` + +## What It Validates + +- Parses `amp.yaml` or `amp.yml` configuration +- Resolves dependencies (respecting `--locked`, `--offline`, `--frozen`) +- Renders Jinja SQL templates with variables +- Validates SELECT-only SQL and incremental constraints +- Infers table schemas via DataFusion planning + +## Notes + +- This command does not write build artifacts (no `sql/` output, no `manifest.json`). diff --git a/crates/bin/ampctl/src/cmd/dataset/package.rs b/crates/bin/ampctl/src/cmd/dataset/package.rs new file mode 100644 index 000000000..432cee362 --- /dev/null +++ b/crates/bin/ampctl/src/cmd/dataset/package.rs @@ -0,0 +1,161 @@ +//! Dataset package command. +//! +//! Creates a deterministic `.tgz` archive from built dataset artifacts. +//! The package can be used for distribution, deployment, or as input +//! to the `dataset register` command. +//! +//! # Package Contents +//! +//! The archive contains: +//! - `manifest.json` - Canonical manifest with file references +//! - `tables/
.sql` - Rendered SQL files +//! - `tables/
.ipc` - Inferred Arrow schemas (IPC format) +//! - `functions/.js` - Function source files (if any) +//! +//! # Determinism +//! +//! Archives are deterministic for identical inputs: +//! - Entries sorted alphabetically by path +//! - Fixed timestamps (Unix epoch) +//! - Consistent permissions (0o644) +//! - No ownership metadata + +use std::path::PathBuf; + +use dataset_authoring::package::PackageBuilder; + +use crate::args::GlobalArgs; + +/// Command-line arguments for the `dataset package` command. +#[derive(Debug, clap::Args)] +#[command( + about = "Package built dataset artifacts into a .tgz archive", + after_help = include_str!("package__after_help.md") +)] +pub struct Args { + #[command(flatten)] + pub global: GlobalArgs, + + /// Directory containing build artifacts (with manifest.json, tables/, etc.). + /// + /// This should be the output directory from `ampctl dataset build`. + /// If not specified, uses the current directory if it contains a manifest.json. + #[arg(long, short = 'd')] + pub dir: Option, + + /// Output path for the package archive. + /// + /// Defaults to `dataset.tgz` in the current directory. + #[arg(long, short = 'o', default_value = "dataset.tgz")] + pub output: PathBuf, +} + +/// Result of a dataset package operation. +#[derive(serde::Serialize)] +struct PackageResult { + status: &'static str, + output: String, + hash: String, + entries: usize, +} + +impl std::fmt::Display for PackageResult { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "Package created: {}", self.output)?; + writeln!(f, " SHA-256: {}", self.hash)?; + writeln!(f, " Entries: {}", self.entries)?; + Ok(()) + } +} + +/// Package built dataset artifacts into a distributable archive. +/// +/// # Errors +/// +/// Returns [`Error`] for missing files, I/O failures, or archive errors. +#[tracing::instrument(skip_all, fields(output = %args.output.display()))] +pub async fn run(args: Args) -> Result<(), Error> { + let Args { + global, + dir, + output, + } = args; + + // Determine the directory to use + let dir = match dir { + Some(d) => d, + None => { + // Auto-detect: use CWD if it contains manifest.json + let cwd = std::env::current_dir().map_err(Error::CurrentDir)?; + let manifest_path = cwd.join("manifest.json"); + if !manifest_path.exists() { + return Err(Error::MissingDirectory); + } + tracing::debug!(cwd = %cwd.display(), "Auto-detected package directory from CWD"); + cwd + } + }; + + // Validate input directory exists + if !dir.is_dir() { + return Err(Error::InvalidDirectory { path: dir }); + } + + // Build the package from directory + tracing::info!("Creating package from {}", dir.display()); + let builder = PackageBuilder::from_directory(&dir).map_err(Error::Package)?; + + // Log what's being included + let entries = builder.entries(); + let entries_len = entries.len(); + tracing::debug!(entries = entries.len(), "Collected package entries"); + + // Write the archive + builder.write_to(&output).map_err(Error::Package)?; + + // Compute hash for verification + let hash = builder.hash().map_err(Error::Package)?; + + tracing::info!(path = %output.display(), hash = %hash, "Package created"); + + let result = PackageResult { + status: "ok", + output: output.display().to_string(), + hash: hash.to_string(), + entries: entries_len, + }; + global.print(&result).map_err(Error::JsonFormattingError)?; + + // List files if verbose logging is enabled + if tracing::enabled!(tracing::Level::DEBUG) { + for entry in entries { + tracing::debug!(" - {} ({} bytes)", entry.path, entry.contents.len()); + } + } + + Ok(()) +} + +/// Errors for dataset package operations. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Failed to get current working directory. + #[error("failed to get current directory")] + CurrentDir(#[source] std::io::Error), + + /// No --dir provided and no manifest.json in current directory. + #[error("no --dir provided and no manifest.json in current directory")] + MissingDirectory, + + /// Invalid directory specified. + #[error("not a directory: {}", path.display())] + InvalidDirectory { path: PathBuf }, + + /// Package assembly failed. + #[error("package assembly failed")] + Package(#[source] dataset_authoring::package::PackageError), + + /// Failed to format JSON for display. + #[error("failed to format package JSON")] + JsonFormattingError(#[source] serde_json::Error), +} diff --git a/crates/bin/ampctl/src/cmd/dataset/package__after_help.md b/crates/bin/ampctl/src/cmd/dataset/package__after_help.md new file mode 100644 index 000000000..78d30b700 --- /dev/null +++ b/crates/bin/ampctl/src/cmd/dataset/package__after_help.md @@ -0,0 +1,56 @@ +## Examples + +Package from current directory (when `manifest.json` is present): +``` +ampctl dataset package +``` + +Package from build directory: +``` +ampctl dataset package --dir build +``` + +Specify output filename: +``` +ampctl dataset package --output my_dataset.tgz +``` + +Full workflow example: +``` +ampctl dataset build --output build +ampctl dataset package --dir build --output my_dataset.tgz +ampctl dataset register my_dataset.tgz +``` + +## Package Contents + +The archive contains all files needed for deployment: + +``` +dataset.tgz/ + manifest.json # Canonical manifest with file refs + sql/ +
.sql # Rendered SQL (SELECT statement) +
.schema.json # Inferred Arrow schema + functions/ + .js # Function sources (if any) +``` + +## Determinism + +The package is built deterministically: +- Files are sorted alphabetically +- Timestamps are set to Unix epoch (0) +- Consistent permissions (0o644 for files, 0o755 for directories) +- No ownership metadata (uid/gid = 0) + +This ensures the same inputs always produce the same package hash. + +## Verification + +The command prints the SHA-256 hash of the package, which can be used +to verify the package was built correctly or check for reproducibility. + +## Structured Output + +Use `--json` to emit machine-readable output for automation. diff --git a/crates/bin/ampctl/src/cmd/dataset/register.rs b/crates/bin/ampctl/src/cmd/dataset/register.rs index b8cf7ed36..3ff24ca5a 100644 --- a/crates/bin/ampctl/src/cmd/dataset/register.rs +++ b/crates/bin/ampctl/src/cmd/dataset/register.rs @@ -1,7 +1,7 @@ //! Dataset registration command. //! //! Registers a dataset by performing the following steps: -//! 1. Loads manifest JSON from local or remote storage +//! 1. Loads manifest JSON from local or remote storage (or converts from package directory) //! 2. Stores the manifest in content-addressable storage (identified by its content hash) //! 3. Links the manifest hash to the specified dataset (namespace/name) //! 4. Tags the dataset revision with a semantic version (if --tag is provided), or updates "dev" by default @@ -12,19 +12,35 @@ //! - **Links to dataset**: Associates the manifest hash with the dataset FQN (namespace/name) //! - **Tags version**: Applies a semantic version tag to identify this dataset revision (optional) //! -//! # Supported Storage Backends +//! # Supported Input Modes //! -//! Manifest files can be loaded from: +//! ## 1. Manifest File (Default) +//! +//! Provide a path or URL to a manifest JSON file directly: //! - Local filesystem: `./manifest.json`, `/path/to/manifest.json` //! - S3: `s3://bucket-name/path/manifest.json` //! - GCS: `gs://bucket-name/path/manifest.json` //! - Azure Blob: `az://container/path/manifest.json` //! - File URL: `file:///path/to/manifest.json` //! +//! ## 2. Package Directory (`--package`) +//! +//! Provide a directory containing the build output from `amp dataset build`: +//! - Reads the authoring `manifest.json` from the directory +//! - Converts it to legacy runtime format using the bridge module +//! - Uploads the converted manifest to the admin API +//! +//! ## 3. Auto-Detection (No File Argument) +//! +//! If no file argument is provided: +//! - Looks for `manifest.json` in the current directory +//! - If found, registers it directly (not as a package) +//! //! # Arguments //! //! - `FQN`: Fully qualified dataset name in format `namespace/name` (e.g., `graph/eth_mainnet`) -//! - `FILE`: Path or URL to the manifest file +//! - `FILE`: Optional path or URL to the manifest file (not used with `--package`) +//! - `--package` / `-p`: Directory containing built dataset package (with manifest.json) //! - `--tag` / `-t`: Optional semantic version tag (e.g., `1.0.0`, `2.1.3`) //! - If not specified, only the "dev" tag is updated to point to this revision //! - Cannot use special tags like "latest" or "dev" - these are managed by the system @@ -34,11 +50,14 @@ //! - Admin URL: `--admin-url` flag or `AMP_ADMIN_URL` env var (default: `http://localhost:1610`) //! - Logging: `AMP_LOG` env var (`error`, `warn`, `info`, `debug`, `trace`) +use std::{fs, path::PathBuf}; + use amp_object_store::{ self as store, ObjectStoreCreationError, ext::{ObjectStoreExt as _, ObjectStoreExtError}, url::{ObjectStoreUrl, ObjectStoreUrlError}, }; +use dataset_authoring::{bridge::LegacyBridge, manifest::AuthoringManifest}; use datasets_common::{fqn::FullyQualifiedName, version::Version}; use monitoring::logging; use object_store::path::Path as ObjectStorePath; @@ -59,8 +78,23 @@ pub struct Args { pub fqn: FullyQualifiedName, /// Path or URL to the manifest file (local path, file://, s3://, gs://, or az://) - #[arg(value_name = "FILE", required = true, value_parser = clap::value_parser!(ManifestFilePath))] - pub manifest_file: ManifestFilePath, + /// + /// Not used with --package. If neither FILE nor --package is provided, + /// looks for manifest.json in the current directory. + #[arg(value_name = "FILE", value_parser = clap::value_parser!(ManifestFilePath), conflicts_with = "package")] + pub manifest_file: Option, + + /// Directory containing built dataset package (from `amp dataset build`) + /// + /// Reads manifest.json from the directory, converts it to legacy runtime + /// format, and uploads to the admin API. Cannot be used with FILE argument. + #[arg( + short = 'p', + long = "package", + value_name = "DIR", + conflicts_with = "manifest_file" + )] + pub package: Option, /// Optional semantic version tag for the dataset (e.g., 1.0.0, 2.1.3) /// Must be a valid semantic version - cannot be "latest" or "dev" @@ -71,38 +105,130 @@ pub struct Args { /// Register a dataset manifest with the admin API. /// -/// Loads manifest content from storage and POSTs to `/datasets` endpoint. +/// Supports three input modes: +/// 1. `--package `: Load authoring manifest from directory, convert to legacy format +/// 2. `FILE` argument: Load manifest from file path or URL +/// 3. Neither: Look for `manifest.json` in current directory /// /// # Errors /// /// Returns [`Error`] for file not found, read failures, invalid paths/URLs, -/// API errors (400/409/500), or network failures. +/// API errors (400/409/500), network failures, or conversion errors. #[tracing::instrument(skip_all, fields(admin_url = %global.admin_url, fqn = %fqn, tag = ?tag))] pub async fn run( Args { global, fqn, manifest_file, + package, tag, }: Args, ) -> Result<(), Error> { - tracing::debug!( - manifest_path = %manifest_file, - fqn = %fqn, - tag = ?tag, - "Loading and registering manifest" - ); - - let manifest_str = load_manifest(&manifest_file).await?; + let manifest_str = if let Some(package_dir) = package { + // Mode 1: Package directory - load authoring manifest and convert to legacy + tracing::debug!( + package_dir = %package_dir.display(), + fqn = %fqn, + tag = ?tag, + "Loading manifest from package directory and converting to legacy format" + ); + load_and_convert_package(&package_dir)? + } else if let Some(ref manifest_path) = manifest_file { + // Mode 2: Manifest file path or URL + tracing::debug!( + manifest_path = %manifest_path, + fqn = %fqn, + tag = ?tag, + "Loading manifest from file" + ); + load_manifest(manifest_path).await? + } else { + // Mode 3: Auto-detect manifest.json in current directory + let cwd = std::env::current_dir().map_err(Error::CurrentDir)?; + let default_manifest = cwd.join("manifest.json"); + if !default_manifest.exists() { + return Err(Error::NoManifestSource); + } + tracing::debug!( + manifest_path = %default_manifest.display(), + fqn = %fqn, + tag = ?tag, + "Loading manifest from current directory" + ); + let manifest_path: ManifestFilePath = + default_manifest.to_string_lossy().parse().map_err(|err| { + Error::InvalidDefaultPath { + path: default_manifest, + source: err, + } + })?; + load_manifest(&manifest_path).await? + }; let manifest: Box = serde_json::from_str(&manifest_str).map_err(Error::InvalidManifest)?; register_manifest(&global, &fqn, tag.as_ref(), manifest).await?; + tracing::info!(fqn = %fqn, tag = ?tag, "Manifest registered successfully"); + println!("Registered manifest for {fqn}"); + if let Some(tag) = &tag { + println!(" Version tag: {tag}"); + } + Ok(()) } +/// Load an authoring manifest from a package directory and convert to legacy format. +/// +/// The package directory should contain `manifest.json` (the authoring manifest) +/// plus `sql/` and optionally `functions/` directories with the referenced files. +fn load_and_convert_package(package_dir: &std::path::Path) -> Result { + // Verify directory exists + if !package_dir.is_dir() { + return Err(Error::PackageNotDirectory { + path: package_dir.to_path_buf(), + }); + } + + // Read authoring manifest + let manifest_path = package_dir.join("manifest.json"); + let manifest_content = fs::read_to_string(&manifest_path).map_err(|err| { + if err.kind() == std::io::ErrorKind::NotFound { + Error::PackageManifestNotFound { + path: manifest_path.clone(), + } + } else { + Error::ReadPackageManifest { + path: manifest_path.clone(), + source: err, + } + } + })?; + + // Parse as authoring manifest + let authoring_manifest: AuthoringManifest = + serde_json::from_str(&manifest_content).map_err(|err| Error::ParseAuthoringManifest { + path: manifest_path, + source: err, + })?; + + // Convert to legacy format using bridge + let bridge = LegacyBridge::new(package_dir); + let legacy_json = bridge + .to_json(&authoring_manifest) + .map_err(|err| Error::BridgeConversion { source: err })?; + + tracing::debug!( + namespace = %authoring_manifest.namespace, + name = %authoring_manifest.name, + tables = authoring_manifest.tables.len(), + "Converted authoring manifest to legacy format" + ); + + Ok(legacy_json) +} + /// Load manifest content from the specified file path or URL. /// /// Uses the object_store crate directly to read the file as UTF-8. @@ -164,6 +290,74 @@ pub async fn register_manifest( /// Errors for manifest registration operations. #[derive(Debug, thiserror::Error)] pub enum Error { + /// No manifest source provided and no manifest.json in current directory. + /// + /// This occurs when neither a FILE argument nor --package flag is provided, + /// and no manifest.json exists in the current working directory. + #[error( + "no manifest source provided; specify FILE, --package, or ensure manifest.json exists in current directory" + )] + NoManifestSource, + + /// Failed to get current directory. + #[error("failed to get current directory")] + CurrentDir(#[source] std::io::Error), + + /// Invalid default manifest path. + /// + /// This occurs when the auto-detected manifest.json path cannot be parsed + /// as a valid object store URL. + #[error("failed to parse default manifest path {}", path.display())] + InvalidDefaultPath { + path: PathBuf, + #[source] + source: ManifestFilePathError, + }, + + /// Package path is not a directory. + /// + /// This occurs when --package is provided but the path does not point to + /// a valid directory. + #[error("package path is not a directory: {}", path.display())] + PackageNotDirectory { path: PathBuf }, + + /// Package manifest.json not found. + /// + /// This occurs when --package is provided but the directory does not + /// contain a manifest.json file. + #[error("manifest.json not found in package directory: {}", path.display())] + PackageManifestNotFound { path: PathBuf }, + + /// Failed to read manifest from package directory. + #[error("failed to read manifest from package: {}", path.display())] + ReadPackageManifest { + path: PathBuf, + #[source] + source: std::io::Error, + }, + + /// Failed to parse authoring manifest. + /// + /// This occurs when the manifest.json in the package directory is not + /// a valid authoring manifest format. + #[error("failed to parse authoring manifest at {}", path.display())] + ParseAuthoringManifest { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + + /// Failed to convert authoring manifest to legacy format. + /// + /// This occurs when the legacy bridge fails to convert the authoring + /// manifest to the runtime format. Common causes include missing SQL + /// or schema files referenced in the manifest. + #[error("failed to convert manifest to legacy format")] + BridgeConversion { + #[source] + source: dataset_authoring::bridge::ToJsonError, + }, + /// Manifest file not found at the specified path #[error("manifest file not found: {path}")] ManifestNotFound { path: String }, diff --git a/crates/core/dataset-authoring/Cargo.toml b/crates/core/dataset-authoring/Cargo.toml new file mode 100644 index 000000000..53be8575e --- /dev/null +++ b/crates/core/dataset-authoring/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "dataset-authoring" +edition.workspace = true +version.workspace = true +license-file.workspace = true + +[dependencies] +async-trait.workspace = true +common = { path = "../common" } +datafusion.workspace = true +datasets-common = { path = "../datasets-common" } +dirs.workspace = true +datasets-derived = { path = "../datasets-derived" } +flate2.workspace = true +minijinja.workspace = true +serde_json_canonicalizer.workspace = true +serde.workspace = true +serde_json.workspace = true +serde_yaml.workspace = true +semver.workspace = true +tar.workspace = true +thiserror.workspace = true +walkdir.workspace = true + +[dev-dependencies] +regex.workspace = true +tempfile.workspace = true +tokio.workspace = true diff --git a/crates/core/dataset-authoring/src/adapter.rs b/crates/core/dataset-authoring/src/adapter.rs new file mode 100644 index 000000000..3af739f63 --- /dev/null +++ b/crates/core/dataset-authoring/src/adapter.rs @@ -0,0 +1,779 @@ +//! Legacy manifest adapter. +//! +//! Converts legacy inline manifest format (from admin API) to the canonical +//! package format used by dataset authoring. +//! +//! # Overview +//! +//! The admin API returns manifests in legacy inline format where SQL, schemas, +//! and function sources are embedded directly in the JSON. The canonical package +//! format uses file references instead: +//! +//! - `tables/
.sql` contains the SQL string +//! - `tables/
.ipc` contains the Arrow IPC binary schema +//! - `functions/.js` contains the function source code +//! - `manifest.json` contains file references with content hashes +//! +//! This module provides the [`LegacyAdapter`] to convert between formats, enabling +//! the cache to store the richer canonical package structure. +//! +//! # Example +//! +//! ```ignore +//! use dataset_authoring::adapter::LegacyAdapter; +//! +//! let legacy_json: serde_json::Value = fetch_from_admin_api().await?; +//! let adapter = LegacyAdapter::new(&cache_dir); +//! let manifest = adapter.adapt_legacy_manifest(legacy_json)?; +//! ``` + +use std::{fs, io, path::Path, sync::Arc}; + +use datasets_common::manifest::TableSchema; +use datasets_derived::manifest::{Manifest as LegacyManifest, TableInput}; + +use crate::{arrow_ipc, files::FileRef}; + +/// Errors that occur during legacy manifest adaptation. +#[derive(Debug, thiserror::Error)] +pub enum AdapterError { + /// Failed to parse legacy manifest JSON. + /// + /// This occurs when the admin API returns JSON that cannot be parsed + /// as a valid legacy manifest structure. + #[error("failed to parse legacy manifest JSON")] + ParseLegacyManifest(#[source] serde_json::Error), + + /// Failed to create tables directory. + /// + /// This occurs when the adapter cannot create the `tables/` directory + /// in the target cache location. + #[error("failed to create tables directory at '{path}'")] + CreateTablesDir { + path: String, + #[source] + source: io::Error, + }, + + /// Failed to create functions directory. + /// + /// This occurs when the adapter cannot create the `functions/` directory + /// in the target cache location. + #[error("failed to create functions directory at '{path}'")] + CreateFunctionsDir { + path: String, + #[source] + source: io::Error, + }, + + /// Failed to write SQL file for a table. + /// + /// This occurs when the adapter cannot write the extracted SQL content + /// to the `tables/
.sql` file. + #[error("failed to write SQL file for table '{table_name}' at '{path}'")] + WriteSqlFile { + table_name: String, + path: String, + #[source] + source: io::Error, + }, + + /// Failed to write IPC schema file for a table. + /// + /// This occurs when the adapter cannot write the converted Arrow IPC + /// schema to the `tables/
.ipc` file. + #[error("failed to write IPC schema file for table '{table_name}' at '{path}'")] + WriteIpcSchemaFile { + table_name: String, + path: String, + #[source] + source: arrow_ipc::IpcSchemaError, + }, + + /// Failed to write function source file. + /// + /// This occurs when the adapter cannot write the extracted function + /// source code to the `functions/.js` file. + #[error("failed to write function source file for function '{func_name}' at '{path}'")] + WriteFunctionFile { + func_name: String, + path: String, + #[source] + source: io::Error, + }, +} + +/// Adapter for converting legacy inline manifests to canonical package format. +/// +/// The adapter extracts inline content from legacy manifests (as received from +/// the admin API) and writes them as separate files in the canonical package +/// structure expected by the cache. +/// +/// # Usage +/// +/// ```ignore +/// let adapter = LegacyAdapter::new("/path/to/cache/hash"); +/// let dep_manifest = adapter.adapt_legacy_manifest(legacy_json)?; +/// ``` +pub struct LegacyAdapter<'a> { + /// Target directory for the canonical package. + target_dir: &'a Path, +} + +impl<'a> LegacyAdapter<'a> { + /// Creates a new legacy adapter with the specified target directory. + /// + /// The target directory is where the canonical package structure will be + /// written (e.g., `~/.amp/registry//`). + pub fn new(target_dir: &'a Path) -> Self { + Self { target_dir } + } + + /// Adapts a legacy manifest JSON to canonical package format. + /// + /// This method: + /// 1. Parses the legacy manifest JSON + /// 2. Extracts inline SQL and writes to `tables/
.sql` + /// 3. Converts inline schemas and writes to `tables/
.ipc` + /// 4. Extracts function sources and writes to `functions/.js` + /// 5. Returns a [`DependencyManifest`](crate::dependency_manifest::DependencyManifest) + /// for use in dependency resolution + /// + /// # Errors + /// + /// Returns [`AdapterError`] if: + /// - The JSON cannot be parsed as a legacy manifest + /// - Required directories cannot be created + /// - Files cannot be written + pub fn adapt_legacy_manifest( + &self, + legacy_json: &serde_json::Value, + ) -> Result { + // Parse the legacy manifest + let legacy: LegacyManifest = serde_json::from_value(legacy_json.clone()) + .map_err(AdapterError::ParseLegacyManifest)?; + + // Extract and write tables + let tables = self.extract_tables(&legacy)?; + + // Extract and write functions + let functions = self.extract_functions(&legacy)?; + + Ok(AdaptedPackage { + kind: legacy.kind.to_string(), + dependencies: legacy.dependencies, + tables, + functions, + }) + } + + /// Extracts tables from legacy manifest and writes files. + fn extract_tables( + &self, + legacy: &LegacyManifest, + ) -> Result< + std::collections::BTreeMap, + AdapterError, + > { + use std::collections::BTreeMap; + + if legacy.tables.is_empty() { + return Ok(BTreeMap::new()); + } + + // Create tables directory + let tables_dir = self.target_dir.join("tables"); + fs::create_dir_all(&tables_dir).map_err(|err| AdapterError::CreateTablesDir { + path: tables_dir.display().to_string(), + source: err, + })?; + + let mut tables = BTreeMap::new(); + + for (table_name, table) in &legacy.tables { + // Extract SQL from View input + let sql_content = match &table.input { + TableInput::View(view) => view.sql.as_str(), + }; + + // Write SQL file + let sql_path = tables_dir.join(format!("{table_name}.sql")); + fs::write(&sql_path, sql_content).map_err(|err| AdapterError::WriteSqlFile { + table_name: table_name.to_string(), + path: sql_path.display().to_string(), + source: err, + })?; + + // Convert inline schema to Arrow SchemaRef and write IPC file + let schema_ref: datafusion::arrow::datatypes::SchemaRef = + table.schema.arrow.clone().into_schema_ref(); + let ipc_path = tables_dir.join(format!("{table_name}.ipc")); + arrow_ipc::write_ipc_schema(&schema_ref, &ipc_path).map_err(|err| { + AdapterError::WriteIpcSchemaFile { + table_name: table_name.to_string(), + path: ipc_path.display().to_string(), + source: err, + } + })?; + + // Create file references with content hashes + let sql_ref = FileRef::from_file(&sql_path, self.target_dir) + .expect("SQL file should exist after writing"); + let ipc_ref = FileRef::from_file(&ipc_path, self.target_dir) + .expect("IPC file should exist after writing"); + + tables.insert( + table_name.clone(), + AdaptedTable { + sql: Some(sql_ref), + ipc: ipc_ref, + network: table.network.clone(), + // Store the original schema for DependencyManifest compatibility + schema: table.schema.clone(), + }, + ); + } + + Ok(tables) + } + + /// Extracts functions from legacy manifest and writes files. + fn extract_functions( + &self, + legacy: &LegacyManifest, + ) -> Result< + std::collections::BTreeMap, + AdapterError, + > { + use std::collections::BTreeMap; + + if legacy.functions.is_empty() { + return Ok(BTreeMap::new()); + } + + // Create functions directory + let functions_dir = self.target_dir.join("functions"); + fs::create_dir_all(&functions_dir).map_err(|err| AdapterError::CreateFunctionsDir { + path: functions_dir.display().to_string(), + source: err, + })?; + + let mut functions = BTreeMap::new(); + + for (func_name, func) in &legacy.functions { + // Write function source file + let func_path = functions_dir.join(&func.source.filename); + fs::write(&func_path, func.source.source.as_ref()).map_err(|err| { + AdapterError::WriteFunctionFile { + func_name: func_name.to_string(), + path: func_path.display().to_string(), + source: err, + } + })?; + + // Create file reference with content hash + let source_ref = FileRef::from_file(&func_path, self.target_dir) + .expect("Function file should exist after writing"); + + functions.insert( + func_name.clone(), + AdaptedFunction { + input_types: func.input_types.clone(), + output_type: func.output_type.clone(), + source: source_ref, + }, + ); + } + + Ok(functions) + } +} + +/// Result of adapting a legacy manifest to canonical package format. +/// +/// Contains both file references (for the package structure) and the original +/// schema data (for dependency resolution compatibility). +#[derive(Debug, Clone)] +pub struct AdaptedPackage { + /// Dataset kind (e.g., "manifest"). + pub kind: String, + /// Dependencies from the legacy manifest. + pub dependencies: std::collections::BTreeMap< + datasets_derived::deps::DepAlias, + datasets_derived::deps::DepReference, + >, + /// Adapted table definitions with file references. + pub tables: std::collections::BTreeMap, + /// Adapted function definitions with file references. + pub functions: + std::collections::BTreeMap, +} + +impl AdaptedPackage { + /// Converts to a [`DependencyManifest`](crate::dependency_manifest::DependencyManifest) + /// for use in dependency resolution. + /// + /// This method creates a `DependencyManifest` from the adapted package, + /// using the original schema data for compatibility with existing resolution logic. + pub fn into_dependency_manifest(self) -> crate::dependency_manifest::DependencyManifest { + use datasets_common::dataset_kind_str::DatasetKindStr; + + use crate::dependency_manifest::{DependencyManifest, DependencyTable}; + + DependencyManifest { + kind: DatasetKindStr::new(self.kind), + dependencies: self.dependencies, + tables: self + .tables + .into_iter() + .map(|(name, table)| { + ( + name, + DependencyTable { + schema: table.schema, + network: table.network, + }, + ) + }) + .collect(), + functions: self + .functions + .into_iter() + .map(|(name, func)| { + ( + name, + datasets_common::manifest::Function { + input_types: func.input_types, + output_type: func.output_type, + source: datasets_common::manifest::FunctionSource { + source: Arc::from(""), + filename: func.source.path.clone(), + }, + }, + ) + }) + .collect(), + } + } +} + +/// Adapted table definition with file references. +#[derive(Debug, Clone)] +pub struct AdaptedTable { + /// Reference to the SQL file (derived tables only). + pub sql: Option, + /// Reference to the IPC schema file. + pub ipc: FileRef, + /// Network for the table. + pub network: datasets_common::network_id::NetworkId, + /// Original schema for DependencyManifest compatibility. + pub schema: TableSchema, +} + +/// Adapted function definition with file reference. +#[derive(Debug, Clone)] +pub struct AdaptedFunction { + /// Function input types. + pub input_types: Vec, + /// Function output type. + pub output_type: datasets_common::manifest::DataType, + /// Reference to the function source file. + pub source: FileRef, +} + +#[cfg(test)] +mod tests { + use datafusion::arrow::datatypes::DataType as ArrowDataType; + use tempfile::TempDir; + + use super::*; + + // ============================================================ + // Helper functions + // ============================================================ + + fn create_legacy_manifest_json() -> serde_json::Value { + serde_json::json!({ + "kind": "manifest", + "dependencies": {}, + "tables": { + "transfers": { + "input": { + "sql": "CREATE TABLE transfers AS SELECT * FROM source.transactions" + }, + "schema": { + "arrow": { + "fields": [ + { + "name": "id", + "type": "UInt64", + "nullable": false + }, + { + "name": "from_address", + "type": "Utf8", + "nullable": true + } + ] + } + }, + "network": "mainnet" + } + }, + "functions": {} + }) + } + + fn create_legacy_manifest_with_function() -> serde_json::Value { + serde_json::json!({ + "kind": "manifest", + "dependencies": {}, + "tables": {}, + "functions": { + "decode": { + "inputTypes": ["Binary"], + "outputType": "Utf8", + "source": { + "source": "function decode(input) { return input.toString(); }", + "filename": "decode.js" + } + } + } + }) + } + + fn create_legacy_manifest_with_dependencies() -> serde_json::Value { + serde_json::json!({ + "kind": "manifest", + "dependencies": { + "eth": "dep_ns/eth_mainnet@b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + }, + "tables": {}, + "functions": {} + }) + } + + // ============================================================ + // LegacyAdapter::new tests + // ============================================================ + + #[test] + fn legacy_adapter_new_creates_instance() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + + //* When + let adapter = LegacyAdapter::new(dir.path()); + + //* Then + assert_eq!(adapter.target_dir, dir.path()); + } + + // ============================================================ + // LegacyAdapter::adapt_legacy_manifest tests - minimal manifest + // ============================================================ + + #[test] + fn adapt_minimal_manifest_succeeds() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let json = serde_json::json!({ + "kind": "manifest", + "dependencies": {}, + "tables": {}, + "functions": {} + }); + let adapter = LegacyAdapter::new(dir.path()); + + //* When + let result = adapter.adapt_legacy_manifest(&json); + + //* Then + let package = result.expect("adaptation should succeed"); + assert_eq!(package.kind, "manifest"); + assert!(package.tables.is_empty()); + assert!(package.functions.is_empty()); + } + + // ============================================================ + // LegacyAdapter::adapt_legacy_manifest tests - tables + // ============================================================ + + #[test] + fn adapt_manifest_extracts_sql_to_file() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let json = create_legacy_manifest_json(); + let adapter = LegacyAdapter::new(dir.path()); + + //* When + let result = adapter.adapt_legacy_manifest(&json); + + //* Then + let package = result.expect("adaptation should succeed"); + assert_eq!(package.tables.len(), 1); + + // Verify SQL file was written + let sql_path = dir.path().join("tables/transfers.sql"); + assert!(sql_path.exists(), "SQL file should exist"); + + let sql_content = fs::read_to_string(&sql_path).expect("should read SQL file"); + assert!( + sql_content.contains("CREATE TABLE transfers"), + "SQL file should contain the query" + ); + } + + #[test] + fn adapt_manifest_extracts_schema_to_ipc() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let json = create_legacy_manifest_json(); + let adapter = LegacyAdapter::new(dir.path()); + + //* When + let result = adapter.adapt_legacy_manifest(&json); + + //* Then + let _package = result.expect("adaptation should succeed"); + + // Verify IPC schema file was written + let ipc_path = dir.path().join("tables/transfers.ipc"); + assert!(ipc_path.exists(), "IPC schema file should exist"); + + // Verify the schema can be read back + let schema_ref = + arrow_ipc::read_ipc_schema(&ipc_path).expect("should read IPC schema file"); + assert_eq!(schema_ref.fields().len(), 2); + assert_eq!(schema_ref.field(0).name(), "id"); + assert_eq!(schema_ref.field(1).name(), "from_address"); + } + + #[test] + fn adapt_manifest_creates_file_refs_with_hashes() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let json = create_legacy_manifest_json(); + let adapter = LegacyAdapter::new(dir.path()); + + //* When + let result = adapter.adapt_legacy_manifest(&json); + + //* Then + let package = result.expect("adaptation should succeed"); + let table = package + .tables + .get(&"transfers".parse().unwrap()) + .expect("should have table"); + + // Verify file references + assert!(table.sql.is_some(), "should have SQL file ref"); + let sql_ref = table.sql.as_ref().unwrap(); + assert_eq!(sql_ref.path, "tables/transfers.sql"); + assert!(!sql_ref.hash.as_str().is_empty(), "should have hash"); + + assert_eq!(table.ipc.path, "tables/transfers.ipc"); + assert!(!table.ipc.hash.as_str().is_empty(), "should have hash"); + } + + #[test] + fn adapt_manifest_preserves_network() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let json = create_legacy_manifest_json(); + let adapter = LegacyAdapter::new(dir.path()); + + //* When + let result = adapter.adapt_legacy_manifest(&json); + + //* Then + let package = result.expect("adaptation should succeed"); + let table = package + .tables + .get(&"transfers".parse().unwrap()) + .expect("should have table"); + + assert_eq!(table.network.to_string(), "mainnet"); + } + + // ============================================================ + // LegacyAdapter::adapt_legacy_manifest tests - functions + // ============================================================ + + #[test] + fn adapt_manifest_extracts_function_source() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let json = create_legacy_manifest_with_function(); + let adapter = LegacyAdapter::new(dir.path()); + + //* When + let result = adapter.adapt_legacy_manifest(&json); + + //* Then + let package = result.expect("adaptation should succeed"); + assert_eq!(package.functions.len(), 1); + + // Verify function file was written + let func_path = dir.path().join("functions/decode.js"); + assert!(func_path.exists(), "function file should exist"); + + let func_content = fs::read_to_string(&func_path).expect("should read function file"); + assert!( + func_content.contains("function decode"), + "function file should contain the source" + ); + } + + #[test] + fn adapt_manifest_preserves_function_types() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let json = create_legacy_manifest_with_function(); + let adapter = LegacyAdapter::new(dir.path()); + + //* When + let result = adapter.adapt_legacy_manifest(&json); + + //* Then + let package = result.expect("adaptation should succeed"); + let func = package + .functions + .get(&"decode".parse().unwrap()) + .expect("should have function"); + + assert_eq!(func.input_types.len(), 1); + assert_eq!(*func.input_types[0].as_arrow(), ArrowDataType::Binary); + assert_eq!(*func.output_type.as_arrow(), ArrowDataType::Utf8); + } + + // ============================================================ + // LegacyAdapter::adapt_legacy_manifest tests - dependencies + // ============================================================ + + #[test] + fn adapt_manifest_preserves_dependencies() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let json = create_legacy_manifest_with_dependencies(); + let adapter = LegacyAdapter::new(dir.path()); + + //* When + let result = adapter.adapt_legacy_manifest(&json); + + //* Then + let package = result.expect("adaptation should succeed"); + assert_eq!(package.dependencies.len(), 1); + assert!(package.dependencies.contains_key(&"eth".parse().unwrap())); + } + + // ============================================================ + // AdaptedPackage::into_dependency_manifest tests + // ============================================================ + + #[test] + fn into_dependency_manifest_converts_correctly() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let json = create_legacy_manifest_json(); + let adapter = LegacyAdapter::new(dir.path()); + let package = adapter + .adapt_legacy_manifest(&json) + .expect("adaptation should succeed"); + + //* When + let dep_manifest = package.into_dependency_manifest(); + + //* Then + assert_eq!(dep_manifest.kind.as_str(), "manifest"); + assert_eq!(dep_manifest.tables.len(), 1); + + let table = dep_manifest + .tables + .get(&"transfers".parse().unwrap()) + .expect("should have table"); + assert_eq!(table.schema.arrow.fields.len(), 2); + assert_eq!(table.network.to_string(), "mainnet"); + } + + // ============================================================ + // Error handling tests + // ============================================================ + + #[test] + fn adapt_manifest_fails_on_invalid_json() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let json = serde_json::json!({ + "invalid": "structure" + }); + let adapter = LegacyAdapter::new(dir.path()); + + //* When + let result = adapter.adapt_legacy_manifest(&json); + + //* Then + assert!(result.is_err(), "should fail on invalid JSON"); + let err = result.expect_err("should have error"); + assert!( + matches!(err, AdapterError::ParseLegacyManifest(_)), + "should be ParseLegacyManifest error" + ); + } + + // ============================================================ + // Round-trip tests (adapter -> bridge -> compare) + // ============================================================ + + #[test] + fn round_trip_table_content_preserved() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let original_sql = "CREATE TABLE transfers AS SELECT * FROM source.transactions"; + let json = serde_json::json!({ + "kind": "manifest", + "dependencies": {}, + "tables": { + "transfers": { + "input": { + "sql": original_sql + }, + "schema": { + "arrow": { + "fields": [ + { + "name": "id", + "type": "UInt64", + "nullable": false + } + ] + } + }, + "network": "mainnet" + } + }, + "functions": {} + }); + + //* When - adapt from legacy + let adapter = LegacyAdapter::new(dir.path()); + let _package = adapter + .adapt_legacy_manifest(&json) + .expect("adaptation should succeed"); + + //* Then - verify SQL content is preserved + let sql_path = dir.path().join("tables/transfers.sql"); + let sql_content = fs::read_to_string(&sql_path).expect("should read SQL file"); + assert_eq!( + sql_content, original_sql, + "SQL content should be preserved exactly" + ); + + // Verify schema is preserved + let ipc_path = dir.path().join("tables/transfers.ipc"); + let schema_ref = arrow_ipc::read_ipc_schema(&ipc_path).expect("should read IPC schema"); + assert_eq!(schema_ref.fields().len(), 1); + assert_eq!(schema_ref.field(0).name(), "id"); + assert!(!schema_ref.field(0).is_nullable()); + } +} diff --git a/crates/core/dataset-authoring/src/arrow_ipc.rs b/crates/core/dataset-authoring/src/arrow_ipc.rs new file mode 100644 index 000000000..288bc5c29 --- /dev/null +++ b/crates/core/dataset-authoring/src/arrow_ipc.rs @@ -0,0 +1,498 @@ +//! Arrow schema serialization in IPC file format. +//! +//! This module handles serialization and deserialization of Arrow schemas to IPC files. +//! The IPC (Inter-Process Communication) file format is Arrow's native binary format, +//! providing efficient, lossless schema storage that preserves all Arrow type information. +//! +//! # Schema File Format +//! +//! Schema files are stored as `tables/
.ipc` and contain an Arrow IPC file +//! with schema metadata only (no record batches). This format: +//! +//! - Is Arrow's native binary format +//! - Preserves all Arrow type information losslessly +//! - Is compact and efficient +//! - Is compatible with Arrow tooling across languages +//! +//! # Example +//! +//! ```ignore +//! use dataset_authoring::arrow_ipc::{write_ipc_schema, read_ipc_schema}; +//! use datafusion::arrow::datatypes::{Schema, Field, DataType}; +//! use std::sync::Arc; +//! +//! // Create a schema +//! let schema = Arc::new(Schema::new(vec![ +//! Field::new("id", DataType::UInt64, false), +//! Field::new("name", DataType::Utf8, true), +//! ])); +//! +//! // Write schema to file +//! write_ipc_schema(&schema, "tables/transfers.ipc")?; +//! +//! // Read schema from file +//! let loaded_schema = read_ipc_schema("tables/transfers.ipc")?; +//! ``` + +use std::{ + fs::File, + io::{self, BufReader, BufWriter}, + path::Path, +}; + +use datafusion::arrow::{ + datatypes::SchemaRef, + ipc::{reader::FileReader, writer::FileWriter}, +}; + +/// Errors that can occur during IPC schema file operations. +#[derive(Debug, thiserror::Error)] +pub enum IpcSchemaError { + /// Failed to read or write the schema file. + #[error("I/O error accessing IPC schema file")] + Io(#[from] io::Error), + + /// Failed to create the IPC file writer. + #[error("failed to create IPC file writer")] + WriterCreation(#[source] datafusion::arrow::error::ArrowError), + + /// Failed to finish writing the IPC file. + #[error("failed to finish writing IPC file")] + WriterFinish(#[source] datafusion::arrow::error::ArrowError), + + /// Failed to create the IPC file reader. + #[error("failed to create IPC file reader")] + ReaderCreation(#[source] datafusion::arrow::error::ArrowError), +} + +/// Writes an Arrow schema to an IPC file. +/// +/// The schema is written as an Arrow IPC file with no record batches. This preserves +/// all Arrow type information losslessly in a compact binary format. +/// +/// # Arguments +/// +/// * `schema` - The Arrow schema to write. +/// * `path` - The path to write the IPC file to. +/// +/// # Errors +/// +/// Returns an error if the file cannot be created or the schema cannot be written. +pub fn write_ipc_schema(schema: &SchemaRef, path: impl AsRef) -> Result<(), IpcSchemaError> { + let file = File::create(path.as_ref())?; + let writer = BufWriter::new(file); + + let mut ipc_writer = + FileWriter::try_new(writer, schema).map_err(IpcSchemaError::WriterCreation)?; + + // Write schema only - no record batches + ipc_writer.finish().map_err(IpcSchemaError::WriterFinish)?; + + Ok(()) +} + +/// Reads an Arrow schema from an IPC file. +/// +/// # Arguments +/// +/// * `path` - The path to read the IPC file from. +/// +/// # Returns +/// +/// The Arrow schema contained in the IPC file. +/// +/// # Errors +/// +/// Returns an error if the file cannot be read or the IPC data is invalid. +pub fn read_ipc_schema(path: impl AsRef) -> Result { + let file = File::open(path.as_ref())?; + let reader = BufReader::new(file); + + let ipc_reader = FileReader::try_new(reader, None).map_err(IpcSchemaError::ReaderCreation)?; + + Ok(ipc_reader.schema()) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; + + use super::*; + + /// Creates a simple test schema with basic types. + fn simple_schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt64, false), + Field::new("name", DataType::Utf8, true), + ])) + } + + /// Creates a schema with complex types. + fn complex_schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("block_num", DataType::UInt64, false), + Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, Some("+00:00".into())), + false, + ), + Field::new("hash", DataType::FixedSizeBinary(32), false), + Field::new("value", DataType::Decimal128(38, 0), true), + ])) + } + + /// Creates a schema with nested types. + fn nested_schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt64, false), + Field::new( + "tags", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + true, + ), + Field::new( + "metadata", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Utf8, true), + ] + .into(), + ), + true, + ), + ])) + } + + mod roundtrip_tests { + use super::*; + + #[test] + fn roundtrips_simple_schema() { + //* Given + let original = simple_schema(); + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("test_simple_schema.ipc"); + + //* When + write_ipc_schema(&original, &file_path).expect("write should succeed"); + let restored = read_ipc_schema(&file_path).expect("read should succeed"); + + //* Then + assert_eq!(original.fields().len(), restored.fields().len()); + for (orig, rest) in original.fields().iter().zip(restored.fields().iter()) { + assert_eq!(orig.name(), rest.name()); + assert_eq!(orig.data_type(), rest.data_type()); + assert_eq!(orig.is_nullable(), rest.is_nullable()); + } + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + + #[test] + fn roundtrips_complex_schema() { + //* Given + let original = complex_schema(); + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("test_complex_schema.ipc"); + + //* When + write_ipc_schema(&original, &file_path).expect("write should succeed"); + let restored = read_ipc_schema(&file_path).expect("read should succeed"); + + //* Then + assert_eq!(original.fields().len(), restored.fields().len()); + for (orig, rest) in original.fields().iter().zip(restored.fields().iter()) { + assert_eq!(orig.name(), rest.name(), "field names should match"); + assert_eq!( + orig.data_type(), + rest.data_type(), + "types should match for field {}", + orig.name() + ); + assert_eq!( + orig.is_nullable(), + rest.is_nullable(), + "nullability should match" + ); + } + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + + #[test] + fn roundtrips_nested_schema() { + //* Given + let original = nested_schema(); + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("test_nested_schema.ipc"); + + //* When + write_ipc_schema(&original, &file_path).expect("write should succeed"); + let restored = read_ipc_schema(&file_path).expect("read should succeed"); + + //* Then + assert_eq!(original.fields().len(), restored.fields().len()); + for (orig, rest) in original.fields().iter().zip(restored.fields().iter()) { + assert_eq!(orig.name(), rest.name(), "field names should match"); + assert_eq!( + orig.data_type(), + rest.data_type(), + "types should match for field {}", + orig.name() + ); + } + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + + #[test] + fn roundtrips_empty_schema() { + //* Given + let original = Arc::new(Schema::empty()); + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("test_empty_schema.ipc"); + + //* When + write_ipc_schema(&original, &file_path).expect("write should succeed"); + let restored = read_ipc_schema(&file_path).expect("read should succeed"); + + //* Then + assert!(restored.fields().is_empty()); + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + + #[test] + fn roundtrips_schema_with_metadata() { + //* Given + let schema = Schema::new(vec![Field::new("id", DataType::UInt64, false)]); + let mut metadata = std::collections::HashMap::new(); + metadata.insert("key1".to_string(), "value1".to_string()); + metadata.insert("key2".to_string(), "value2".to_string()); + let original = Arc::new(schema.with_metadata(metadata)); + + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("test_schema_with_metadata.ipc"); + + //* When + write_ipc_schema(&original, &file_path).expect("write should succeed"); + let restored = read_ipc_schema(&file_path).expect("read should succeed"); + + //* Then + assert_eq!(original.metadata(), restored.metadata()); + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + } + + mod error_tests { + use super::*; + + #[test] + fn returns_error_for_nonexistent_file() { + //* Given + let path = "/nonexistent/path/to/schema.ipc"; + + //* When + let result = read_ipc_schema(path); + + //* Then + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), IpcSchemaError::Io(_))); + } + + #[test] + fn returns_error_for_invalid_ipc_file() { + //* Given + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("invalid_schema.ipc"); + std::fs::write(&file_path, b"not a valid IPC file").expect("write should succeed"); + + //* When + let result = read_ipc_schema(&file_path); + + //* Then + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + IpcSchemaError::ReaderCreation(_) + )); + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + + #[test] + fn returns_error_for_unwritable_path() { + //* Given + let schema = simple_schema(); + let path = "/nonexistent/directory/schema.ipc"; + + //* When + let result = write_ipc_schema(&schema, path); + + //* Then + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), IpcSchemaError::Io(_))); + } + } + + mod file_format_tests { + use super::*; + + #[test] + fn writes_valid_ipc_file() { + //* Given + let schema = simple_schema(); + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("test_valid_ipc.ipc"); + + //* When + write_ipc_schema(&schema, &file_path).expect("write should succeed"); + let contents = std::fs::read(&file_path).expect("read should succeed"); + + //* Then + // IPC files start with "ARROW1" magic bytes + assert!(contents.len() >= 6, "file should have magic bytes"); + assert_eq!(&contents[0..6], b"ARROW1", "should have IPC magic bytes"); + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + + #[test] + fn file_contains_no_record_batches() { + //* Given + let schema = simple_schema(); + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("test_no_batches.ipc"); + + //* When + write_ipc_schema(&schema, &file_path).expect("write should succeed"); + + let file = File::open(&file_path).expect("open should succeed"); + let reader = FileReader::try_new(BufReader::new(file), None) + .expect("reader creation should succeed"); + + //* Then + assert_eq!(reader.num_batches(), 0, "should have no record batches"); + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + } + + mod type_coverage_tests { + use super::*; + + #[test] + fn roundtrips_all_primitive_types() { + //* Given + let original = Arc::new(Schema::new(vec![ + Field::new("bool", DataType::Boolean, false), + Field::new("int8", DataType::Int8, false), + Field::new("int16", DataType::Int16, false), + Field::new("int32", DataType::Int32, false), + Field::new("int64", DataType::Int64, false), + Field::new("uint8", DataType::UInt8, false), + Field::new("uint16", DataType::UInt16, false), + Field::new("uint32", DataType::UInt32, false), + Field::new("uint64", DataType::UInt64, false), + Field::new("float16", DataType::Float16, false), + Field::new("float32", DataType::Float32, false), + Field::new("float64", DataType::Float64, false), + Field::new("utf8", DataType::Utf8, true), + Field::new("large_utf8", DataType::LargeUtf8, true), + Field::new("binary", DataType::Binary, true), + Field::new("large_binary", DataType::LargeBinary, true), + ])); + + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("test_primitive_types.ipc"); + + //* When + write_ipc_schema(&original, &file_path).expect("write should succeed"); + let restored = read_ipc_schema(&file_path).expect("read should succeed"); + + //* Then + assert_eq!(original.fields().len(), restored.fields().len()); + for (orig, rest) in original.fields().iter().zip(restored.fields().iter()) { + assert_eq!( + orig.data_type(), + rest.data_type(), + "type mismatch for field {}", + orig.name() + ); + } + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + + #[test] + fn roundtrips_temporal_types() { + //* Given + let original = Arc::new(Schema::new(vec![ + Field::new("date32", DataType::Date32, false), + Field::new("date64", DataType::Date64, false), + Field::new("time32_sec", DataType::Time32(TimeUnit::Second), false), + Field::new( + "time32_milli", + DataType::Time32(TimeUnit::Millisecond), + false, + ), + Field::new( + "time64_micro", + DataType::Time64(TimeUnit::Microsecond), + false, + ), + Field::new("time64_nano", DataType::Time64(TimeUnit::Nanosecond), false), + Field::new( + "timestamp_utc", + DataType::Timestamp(TimeUnit::Nanosecond, Some("+00:00".into())), + false, + ), + Field::new( + "timestamp_none", + DataType::Timestamp(TimeUnit::Microsecond, None), + false, + ), + Field::new("duration_sec", DataType::Duration(TimeUnit::Second), false), + Field::new( + "interval_month", + DataType::Interval(datafusion::arrow::datatypes::IntervalUnit::YearMonth), + false, + ), + ])); + + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("test_temporal_types.ipc"); + + //* When + write_ipc_schema(&original, &file_path).expect("write should succeed"); + let restored = read_ipc_schema(&file_path).expect("read should succeed"); + + //* Then + assert_eq!(original.fields().len(), restored.fields().len()); + for (orig, rest) in original.fields().iter().zip(restored.fields().iter()) { + assert_eq!( + orig.data_type(), + rest.data_type(), + "type mismatch for field {}", + orig.name() + ); + } + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + } +} diff --git a/crates/core/dataset-authoring/src/arrow_json.rs b/crates/core/dataset-authoring/src/arrow_json.rs new file mode 100644 index 000000000..5f5fb610f --- /dev/null +++ b/crates/core/dataset-authoring/src/arrow_json.rs @@ -0,0 +1,479 @@ +//! Arrow schema serialization in JSON format. +//! +//! This module handles serialization and deserialization of Arrow schemas to JSON files. +//! The format used is compatible with the existing Amp manifest schema format, which +//! represents Arrow types using DataFusion's serde implementation. +//! +//! **Note**: This module is primarily used by the legacy bridge for converting between +//! the canonical IPC format and the legacy JSON format expected by the admin API. +//! New code should prefer using `arrow_ipc` for native Arrow IPC schema I/O. +//! +//! # Schema File Format +//! +//! Schema files in JSON format contain: +//! +//! ```json +//! { +//! "fields": [ +//! { "name": "column_name", "type": "UInt64", "nullable": false }, +//! { "name": "other_col", "type": { "Timestamp": ["Nanosecond", "+00:00"] }, "nullable": true } +//! ] +//! } +//! ``` +//! +//! This format: +//! - Uses DataFusion's built-in serde for Arrow `DataType` +//! - Is compatible with the legacy manifest bridge +//! - Supports all Arrow types used by Amp datasets +//! +//! # Example +//! +//! ```ignore +//! use dataset_authoring::arrow_json::{write_schema_file, read_schema_file}; +//! use datasets_common::manifest::TableSchema; +//! +//! // Write schema to file (legacy JSON format for admin API) +//! write_schema_file(&schema, "legacy_schema.json")?; +//! +//! // Read schema from file +//! let loaded_schema = read_schema_file("legacy_schema.json")?; +//! ``` + +use std::{ + io::{self, BufReader, BufWriter}, + path::Path, +}; + +use datasets_common::manifest::ArrowSchema; + +/// Errors that can occur during schema file operations. +#[derive(Debug, thiserror::Error)] +pub enum SchemaFileError { + /// Failed to read or write the schema file. + #[error("I/O error accessing schema file")] + Io(#[from] io::Error), + + /// Failed to serialize schema to JSON. + #[error("failed to serialize schema to JSON")] + Serialize(#[source] serde_json::Error), + + /// Failed to deserialize schema from JSON. + #[error("failed to deserialize schema from JSON")] + Deserialize(#[source] serde_json::Error), +} + +/// Writes an Arrow schema to a JSON file. +/// +/// The schema is written using the Amp manifest format, which is compatible +/// with DataFusion's serde implementation for Arrow types. +/// +/// # Arguments +/// +/// * `schema` - The Arrow schema to write. +/// * `path` - The path to write the schema file to. +/// +/// # Errors +/// +/// Returns an error if the file cannot be written or the schema cannot be serialized. +pub fn write_schema_file( + schema: &ArrowSchema, + path: impl AsRef, +) -> Result<(), SchemaFileError> { + let file = std::fs::File::create(path.as_ref())?; + let writer = BufWriter::new(file); + + serde_json::to_writer_pretty(writer, schema).map_err(SchemaFileError::Serialize)?; + + Ok(()) +} + +/// Reads an Arrow schema from a JSON file. +/// +/// # Arguments +/// +/// * `path` - The path to read the schema file from. +/// +/// # Returns +/// +/// The deserialized Arrow schema. +/// +/// # Errors +/// +/// Returns an error if the file cannot be read or the JSON is invalid. +pub fn read_schema_file(path: impl AsRef) -> Result { + let file = std::fs::File::open(path.as_ref())?; + let reader = BufReader::new(file); + + serde_json::from_reader(reader).map_err(SchemaFileError::Deserialize) +} + +/// Serializes an Arrow schema to a JSON string. +/// +/// This is useful for generating schema JSON for embedding in manifests +/// or for testing purposes. +/// +/// # Arguments +/// +/// * `schema` - The Arrow schema to serialize. +/// +/// # Returns +/// +/// The JSON representation of the schema. +pub fn schema_to_json(schema: &ArrowSchema) -> Result { + serde_json::to_string_pretty(schema).map_err(SchemaFileError::Serialize) +} + +/// Deserializes an Arrow schema from a JSON string. +/// +/// # Arguments +/// +/// * `json` - The JSON string to parse. +/// +/// # Returns +/// +/// The deserialized Arrow schema. +pub fn schema_from_json(json: &str) -> Result { + serde_json::from_str(json).map_err(SchemaFileError::Deserialize) +} + +#[cfg(test)] +mod tests { + use datafusion::arrow::datatypes::DataType; + use datasets_common::manifest::Field; + + use super::*; + + /// Creates a simple test schema with basic types. + fn simple_schema() -> ArrowSchema { + ArrowSchema { + fields: vec![ + Field { + name: "id".to_string(), + type_: DataType::UInt64.into(), + nullable: false, + }, + Field { + name: "name".to_string(), + type_: DataType::Utf8.into(), + nullable: true, + }, + ], + } + } + + /// Creates a schema with complex types. + fn complex_schema() -> ArrowSchema { + ArrowSchema { + fields: vec![ + Field { + name: "block_num".to_string(), + type_: DataType::UInt64.into(), + nullable: false, + }, + Field { + name: "timestamp".to_string(), + type_: DataType::Timestamp( + datafusion::arrow::datatypes::TimeUnit::Nanosecond, + Some("+00:00".into()), + ) + .into(), + nullable: false, + }, + Field { + name: "hash".to_string(), + type_: DataType::FixedSizeBinary(32).into(), + nullable: false, + }, + Field { + name: "value".to_string(), + type_: DataType::Decimal128(38, 0).into(), + nullable: true, + }, + ], + } + } + + mod serialization_tests { + use super::*; + + #[test] + fn serializes_simple_schema() { + //* Given + let schema = simple_schema(); + + //* When + let json = schema_to_json(&schema).expect("serialization should succeed"); + + //* Then + assert!( + json.contains(r#""name": "id""#), + "should contain field name" + ); + assert!(json.contains(r#""type": "UInt64""#), "should contain type"); + assert!( + json.contains(r#""nullable": false"#), + "should contain nullable" + ); + } + + #[test] + fn serializes_complex_types() { + //* Given + let schema = complex_schema(); + + //* When + let json = schema_to_json(&schema).expect("serialization should succeed"); + + //* Then + // Check Timestamp type is serialized correctly + assert!( + json.contains("Timestamp"), + "should contain Timestamp type name" + ); + assert!(json.contains("Nanosecond"), "should contain time unit"); + + // Check FixedSizeBinary + assert!( + json.contains("FixedSizeBinary"), + "should contain FixedSizeBinary" + ); + assert!(json.contains("32"), "should contain binary size"); + + // Check Decimal128 + assert!(json.contains("Decimal128"), "should contain Decimal128"); + assert!(json.contains("38"), "should contain precision"); + } + } + + mod roundtrip_tests { + use super::*; + + #[test] + fn roundtrips_simple_schema() { + //* Given + let original = simple_schema(); + + //* When + let json = schema_to_json(&original).expect("serialization should succeed"); + let restored = schema_from_json(&json).expect("deserialization should succeed"); + + //* Then + assert_eq!(original.fields.len(), restored.fields.len()); + for (orig, rest) in original.fields.iter().zip(restored.fields.iter()) { + assert_eq!(orig.name, rest.name); + assert_eq!(orig.nullable, rest.nullable); + assert_eq!(*orig.type_.as_arrow(), *rest.type_.as_arrow()); + } + } + + #[test] + fn roundtrips_complex_schema() { + //* Given + let original = complex_schema(); + + //* When + let json = schema_to_json(&original).expect("serialization should succeed"); + let restored = schema_from_json(&json).expect("deserialization should succeed"); + + //* Then + assert_eq!(original.fields.len(), restored.fields.len()); + for (orig, rest) in original.fields.iter().zip(restored.fields.iter()) { + assert_eq!(orig.name, rest.name, "field names should match"); + assert_eq!(orig.nullable, rest.nullable, "nullability should match"); + assert_eq!( + *orig.type_.as_arrow(), + *rest.type_.as_arrow(), + "types should match for field {}", + orig.name + ); + } + } + + #[test] + fn roundtrips_empty_schema() { + //* Given + let original = ArrowSchema { fields: vec![] }; + + //* When + let json = schema_to_json(&original).expect("serialization should succeed"); + let restored = schema_from_json(&json).expect("deserialization should succeed"); + + //* Then + assert!(restored.fields.is_empty()); + } + } + + mod file_io_tests { + use std::io::Write; + + use super::*; + + #[test] + fn writes_and_reads_schema_file() { + //* Given + let schema = simple_schema(); + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("test_schema.json"); + + //* When + write_schema_file(&schema, &file_path).expect("write should succeed"); + let loaded = read_schema_file(&file_path).expect("read should succeed"); + + //* Then + assert_eq!(schema.fields.len(), loaded.fields.len()); + for (orig, rest) in schema.fields.iter().zip(loaded.fields.iter()) { + assert_eq!(orig.name, rest.name); + assert_eq!(orig.nullable, rest.nullable); + assert_eq!(*orig.type_.as_arrow(), *rest.type_.as_arrow()); + } + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + + #[test] + fn returns_error_for_nonexistent_file() { + //* Given + let path = "/nonexistent/path/to/schema.json"; + + //* When + let result = read_schema_file(path); + + //* Then + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), SchemaFileError::Io(_))); + } + + #[test] + fn returns_error_for_invalid_json() { + //* Given + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("invalid_schema.json"); + let mut file = std::fs::File::create(&file_path).expect("file creation should succeed"); + file.write_all(b"{ invalid json }") + .expect("write should succeed"); + + //* When + let result = read_schema_file(&file_path); + + //* Then + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + SchemaFileError::Deserialize(_) + )); + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + + #[test] + fn produces_pretty_formatted_output() { + //* Given + let schema = simple_schema(); + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("pretty_schema.json"); + + //* When + write_schema_file(&schema, &file_path).expect("write should succeed"); + let content = std::fs::read_to_string(&file_path).expect("read should succeed"); + + //* Then + // Pretty printed JSON should have newlines and indentation + assert!(content.contains('\n'), "should have newlines"); + assert!(content.contains(" "), "should have indentation"); + + // Cleanup + let _ = std::fs::remove_file(&file_path); + } + } + + mod known_schemas_tests { + use datafusion::arrow::datatypes::{ + DataType as ArrowDataType, Field as ArrowField, Schema, + }; + + use super::*; + + /// Tests that our format matches what's expected for integration with + /// existing manifests. + #[test] + fn matches_expected_manifest_format() { + //* Given + let schema = ArrowSchema { + fields: vec![ + Field { + name: "_block_num".to_string(), + type_: ArrowDataType::UInt64.into(), + nullable: false, + }, + Field { + name: "tx_hash".to_string(), + type_: ArrowDataType::FixedSizeBinary(32).into(), + nullable: false, + }, + ], + }; + + //* When + let json = schema_to_json(&schema).expect("serialization should succeed"); + + //* Then + // Verify the format matches expected manifest JSON structure + let parsed: serde_json::Value = + serde_json::from_str(&json).expect("should be valid JSON"); + + assert!(parsed.get("fields").is_some(), "should have fields array"); + + let fields = parsed["fields"].as_array().expect("fields should be array"); + assert_eq!(fields.len(), 2); + + // First field + assert_eq!(fields[0]["name"], "_block_num"); + assert_eq!(fields[0]["type"], "UInt64"); + assert_eq!(fields[0]["nullable"], false); + + // Second field with complex type + assert_eq!(fields[1]["name"], "tx_hash"); + assert!( + fields[1]["type"].is_object(), + "complex type should be object" + ); + assert_eq!(fields[1]["type"]["FixedSizeBinary"], 32); + } + + #[test] + fn converts_from_datafusion_schema() { + //* Given + let df_schema = Schema::new(vec![ + ArrowField::new("id", ArrowDataType::Int64, false), + ArrowField::new("value", ArrowDataType::Float64, true), + ]); + + //* When + let arrow_schema = ArrowSchema { + fields: df_schema + .fields() + .iter() + .map(|f| Field { + name: f.name().clone(), + type_: f.data_type().clone().into(), + nullable: f.is_nullable(), + }) + .collect(), + }; + let json = schema_to_json(&arrow_schema).expect("serialization should succeed"); + let restored = schema_from_json(&json).expect("deserialization should succeed"); + + //* Then + assert_eq!(restored.fields.len(), 2); + assert_eq!(restored.fields[0].name, "id"); + assert_eq!(*restored.fields[0].type_.as_arrow(), ArrowDataType::Int64); + assert!(!restored.fields[0].nullable); + + assert_eq!(restored.fields[1].name, "value"); + assert_eq!(*restored.fields[1].type_.as_arrow(), ArrowDataType::Float64); + assert!(restored.fields[1].nullable); + } + } +} diff --git a/crates/core/dataset-authoring/src/bridge.rs b/crates/core/dataset-authoring/src/bridge.rs new file mode 100644 index 000000000..14581ced3 --- /dev/null +++ b/crates/core/dataset-authoring/src/bridge.rs @@ -0,0 +1,1114 @@ +//! Legacy manifest bridge. +//! +//! Converts the new file-reference based authoring manifest to the legacy inline +//! manifest format expected by existing Amp instances. +//! +//! # Overview +//! +//! The authoring workflow uses file references in manifests: +//! - SQL files at `tables/
.sql` (derived tables only) +//! - IPC schema files at `tables/
.ipc` +//! - Function source files at `functions/.js` +//! +//! The legacy runtime format requires inline content: +//! - `tables..input.view.sql` contains the SQL string +//! - `tables..schema` contains the Arrow schema +//! - `functions..source` contains `{ source: , filename: }` +//! +//! This module provides the [`LegacyBridge`] type to convert between formats. +//! +//! Note: Raw tables (no SQL definition) are not supported by the legacy bridge. +//! +//! # Example +//! +//! ```ignore +//! use dataset_authoring::bridge::LegacyBridge; +//! +//! let bridge = LegacyBridge::new(project_dir); +//! let legacy_manifest = bridge.convert(&authoring_manifest)?; +//! let json = serde_json::to_string_pretty(&legacy_manifest)?; +//! ``` + +use std::{collections::BTreeMap, fs, io, path::Path, sync::Arc}; + +use datasets_common::manifest::{FunctionSource, TableSchema}; +use datasets_derived::{ + DerivedDatasetKind, + deps::{DepReference, HashOrVersion}, + func_name::FuncName, + manifest::{Function, Manifest, Table, TableInput, View}, +}; + +use crate::{arrow_ipc, manifest::AuthoringManifest}; + +/// Errors that occur during legacy bridge conversion. +#[derive(Debug, thiserror::Error)] +pub enum BridgeError { + /// Failed to read SQL file for a table. + /// + /// This occurs when the SQL file referenced in the manifest cannot be read + /// from disk. The file should exist at the path specified in the manifest's + /// table definition. + #[error("failed to read SQL file for table '{table_name}' at path '{path}'")] + ReadSqlFile { + /// The table name. + table_name: String, + /// The path to the SQL file. + path: String, + /// The underlying I/O error. + #[source] + source: io::Error, + }, + + /// Failed to read IPC schema file for a table. + /// + /// This occurs when the IPC schema file referenced in the manifest cannot + /// be read from disk. The file should exist at the path specified in the + /// manifest's table definition. + #[error("failed to read IPC schema file for table '{table_name}' at path '{path}'")] + ReadIpcSchemaFile { + /// The table name. + table_name: String, + /// The path to the IPC schema file. + path: String, + /// The underlying IPC schema error. + #[source] + source: arrow_ipc::IpcSchemaError, + }, + + /// Failed to read function source file. + /// + /// This occurs when the function source file referenced in the manifest + /// cannot be read from disk. The file should exist at the path specified + /// in the manifest's function definition. + #[error("failed to read function source file for function '{func_name}' at path '{path}'")] + ReadFunctionFile { + /// The function name. + func_name: FuncName, + /// The path to the function source file. + path: String, + /// The underlying I/O error. + #[source] + source: io::Error, + }, + + /// Missing SQL file reference for derived table. + /// + /// This occurs when a derived table is missing a SQL file reference. + /// Raw tables (which have no SQL) cannot be converted to the legacy format. + #[error( + "table '{table_name}' has no SQL definition (raw tables not supported in legacy bridge)" + )] + MissingSql { + /// The table name. + table_name: String, + }, + + /// SQL content is invalid. + /// + /// This occurs when the SQL content read from the file cannot be parsed + /// as a valid SQL string for the legacy manifest. + #[error("invalid SQL content for table '{table_name}'")] + InvalidSql { + /// The table name. + table_name: String, + /// The underlying parse error. + #[source] + source: datasets_derived::sql_str::SqlStrError, + }, +} + +/// Bridge for converting authoring manifests to legacy runtime format. +/// +/// The legacy bridge reads file content from disk and inlines it into the +/// runtime manifest structure expected by existing Amp instances. +/// +/// # Usage +/// +/// ```ignore +/// let bridge = LegacyBridge::new("/path/to/project"); +/// let legacy = bridge.convert(&authoring_manifest)?; +/// ``` +pub struct LegacyBridge<'a> { + /// Base path for resolving file references. + base_path: &'a Path, +} + +impl<'a> LegacyBridge<'a> { + /// Creates a new legacy bridge with the specified base path. + /// + /// The base path is used to resolve relative file paths in the authoring + /// manifest (e.g., `tables/transfers.sql` becomes `/tables/transfers.sql`). + pub fn new(base_path: &'a Path) -> Self { + Self { base_path } + } + + /// Converts an authoring manifest to the legacy runtime format. + /// + /// This method: + /// 1. Reads SQL files and inlines them into table definitions + /// 2. Reads schema files and converts them to `TableSchema` + /// 3. Reads function source files and inlines them + /// + /// # Errors + /// + /// Returns [`BridgeError`] if any file cannot be read or parsed. + pub fn convert(&self, manifest: &AuthoringManifest) -> Result { + let tables = self.convert_tables(manifest)?; + let functions = self.convert_functions(manifest)?; + + Ok(Manifest { + kind: DerivedDatasetKind, + start_block: None, + dependencies: manifest + .dependencies + .iter() + .map(|(alias, info)| { + // Convert DepInfo to DepReference format (namespace/name@hash) + let reference = DepReference::new( + info.namespace.clone(), + info.name.clone(), + HashOrVersion::Hash(info.hash.clone()), + ); + (alias.clone(), reference) + }) + .collect(), + tables, + functions, + }) + } + + /// Converts authoring table definitions to legacy format. + fn convert_tables( + &self, + manifest: &AuthoringManifest, + ) -> Result, BridgeError> { + let mut tables = BTreeMap::new(); + + for (table_name, table_def) in &manifest.tables { + // Read SQL file (required for derived tables) + let sql_ref = table_def + .sql + .as_ref() + .ok_or_else(|| BridgeError::MissingSql { + table_name: table_name.to_string(), + })?; + let sql_path = self.base_path.join(&sql_ref.path); + let sql_content = + fs::read_to_string(&sql_path).map_err(|err| BridgeError::ReadSqlFile { + table_name: table_name.to_string(), + path: sql_ref.path.clone(), + source: err, + })?; + + // Parse SQL content + let sql_str = sql_content.parse().map_err(|err| BridgeError::InvalidSql { + table_name: table_name.to_string(), + source: err, + })?; + + // Read IPC schema file and convert to legacy ArrowSchema format + let ipc_path = self.base_path.join(&table_def.ipc.path); + let schema_ref = arrow_ipc::read_ipc_schema(&ipc_path).map_err(|err| { + BridgeError::ReadIpcSchemaFile { + table_name: table_name.to_string(), + path: table_def.ipc.path.clone(), + source: err, + } + })?; + let arrow_schema = datasets_common::manifest::ArrowSchema::from(&schema_ref); + + let table = Table { + input: TableInput::View(View { sql: sql_str }), + schema: TableSchema { + arrow: arrow_schema, + }, + network: table_def.network.clone(), + }; + + tables.insert(table_name.clone(), table); + } + + Ok(tables) + } + + /// Converts authoring function definitions to legacy format. + fn convert_functions( + &self, + manifest: &AuthoringManifest, + ) -> Result, BridgeError> { + let mut functions = BTreeMap::new(); + + for (func_name, func_def) in &manifest.functions { + // Read function source file + let source_path = self.base_path.join(&func_def.source.path); + let source_content = + fs::read_to_string(&source_path).map_err(|err| BridgeError::ReadFunctionFile { + func_name: func_name.clone(), + path: func_def.source.path.clone(), + source: err, + })?; + + // Extract filename from path for the legacy format + let filename = Path::new(&func_def.source.path) + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or(&func_def.source.path) + .to_string(); + + let function = Function { + input_types: func_def.input_types.clone(), + output_type: func_def.output_type.clone(), + source: FunctionSource { + source: Arc::from(source_content.as_str()), + filename, + }, + }; + + functions.insert(func_name.clone(), function); + } + + Ok(functions) + } + + /// Converts an authoring manifest to a JSON string in legacy format. + /// + /// This is a convenience method that converts the manifest and serializes + /// it to pretty-printed JSON suitable for upload to the legacy admin API. + /// + /// # Errors + /// + /// Returns [`BridgeError`] if conversion fails, or serialization error + /// wrapped in the result if JSON serialization fails. + pub fn to_json(&self, manifest: &AuthoringManifest) -> Result { + let legacy = self.convert(manifest).map_err(ToJsonError::Bridge)?; + serde_json::to_string_pretty(&legacy).map_err(ToJsonError::Serialize) + } +} + +/// Errors that occur during JSON conversion. +#[derive(Debug, thiserror::Error)] +pub enum ToJsonError { + /// Failed to convert manifest to legacy format. + #[error("failed to convert manifest to legacy format")] + Bridge(#[source] BridgeError), + + /// Failed to serialize legacy manifest to JSON. + #[error("failed to serialize legacy manifest to JSON")] + Serialize(#[source] serde_json::Error), +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, sync::Arc}; + + use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field as ArrowField, Schema}; + use datasets_common::{hash::Hash, manifest::DataType}; + use tempfile::TempDir; + + use super::*; + use crate::{ + arrow_ipc::write_ipc_schema, + files::FileRef, + manifest::{AuthoringFunctionDef, AuthoringManifest, DepInfo, TableDef}, + }; + + // ============================================================ + // Helper functions + // ============================================================ + + fn create_test_hash() -> Hash { + "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash") + } + + fn create_minimal_manifest() -> AuthoringManifest { + AuthoringManifest { + namespace: "test_ns".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + kind: "derived".to_string(), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + } + } + + fn setup_test_files(dir: &TempDir) { + let tables_dir = dir.path().join("tables"); + fs::create_dir_all(&tables_dir).expect("should create tables dir"); + + // Create SQL file + fs::write( + tables_dir.join("transfers.sql"), + "CREATE TABLE transfers AS SELECT * FROM source.transactions", + ) + .expect("should write sql"); + + // Create IPC schema file + let schema = Arc::new(Schema::new(vec![ArrowField::new( + "id", + ArrowDataType::UInt64, + false, + )])); + write_ipc_schema(&schema, tables_dir.join("transfers.ipc")) + .expect("should write ipc schema"); + } + + fn setup_function_files(dir: &TempDir) { + let functions_dir = dir.path().join("functions"); + fs::create_dir_all(&functions_dir).expect("should create functions dir"); + + fs::write( + functions_dir.join("decode.js"), + "function decode(input) { return input.toString(); }", + ) + .expect("should write function"); + } + + // ============================================================ + // LegacyBridge::new tests + // ============================================================ + + #[test] + fn legacy_bridge_new_creates_instance() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + + //* When + let bridge = LegacyBridge::new(dir.path()); + + //* Then + assert_eq!(bridge.base_path, dir.path()); + } + + // ============================================================ + // LegacyBridge::convert tests - minimal manifest + // ============================================================ + + #[test] + fn convert_minimal_manifest_succeeds() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let manifest = create_minimal_manifest(); + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.convert(&manifest); + + //* Then + let legacy = result.expect("conversion should succeed"); + assert!(legacy.tables.is_empty()); + assert!(legacy.functions.is_empty()); + } + + // ============================================================ + // LegacyBridge::convert tests - tables + // ============================================================ + + #[test] + fn convert_manifest_with_table_inlines_sql() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + setup_test_files(&dir); + + let mut manifest = create_minimal_manifest(); + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + create_test_hash(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), create_test_hash()), + network: "mainnet".parse().expect("valid network"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.convert(&manifest); + + //* Then + let legacy = result.expect("conversion should succeed"); + assert_eq!(legacy.tables.len(), 1); + + let table = legacy + .tables + .get(&"transfers".parse().unwrap()) + .expect("should have table"); + + match &table.input { + TableInput::View(view) => { + assert!(view.sql.as_str().contains("CREATE TABLE transfers")); + } + } + } + + #[test] + fn convert_manifest_with_table_reads_schema() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + setup_test_files(&dir); + + let mut manifest = create_minimal_manifest(); + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + create_test_hash(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), create_test_hash()), + network: "mainnet".parse().expect("valid network"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.convert(&manifest); + + //* Then + let legacy = result.expect("conversion should succeed"); + let table = legacy + .tables + .get(&"transfers".parse().unwrap()) + .expect("should have table"); + + assert_eq!(table.schema.arrow.fields.len(), 1); + assert_eq!(table.schema.arrow.fields[0].name, "id"); + } + + #[test] + fn convert_manifest_preserves_network() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + setup_test_files(&dir); + + let mut manifest = create_minimal_manifest(); + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + create_test_hash(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), create_test_hash()), + network: "arbitrum-one".parse().expect("valid network"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.convert(&manifest); + + //* Then + let legacy = result.expect("conversion should succeed"); + let table = legacy + .tables + .get(&"transfers".parse().unwrap()) + .expect("should have table"); + + assert_eq!(table.network.to_string(), "arbitrum-one"); + } + + #[test] + fn convert_manifest_fails_on_missing_sql_file() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + // Don't create the sql file + + let mut manifest = create_minimal_manifest(); + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + create_test_hash(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), create_test_hash()), + network: "mainnet".parse().expect("valid network"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.convert(&manifest); + + //* Then + assert!(result.is_err()); + let err = result.expect_err("should fail"); + assert!( + matches!(err, BridgeError::ReadSqlFile { .. }), + "should be ReadSqlFile error, got: {:?}", + err + ); + } + + #[test] + fn convert_manifest_fails_on_missing_schema_file() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let tables_dir = dir.path().join("tables"); + fs::create_dir_all(&tables_dir).expect("should create tables dir"); + fs::write( + tables_dir.join("transfers.sql"), + "CREATE TABLE transfers AS SELECT 1", + ) + .expect("should write sql"); + // Don't create the IPC schema file + + let mut manifest = create_minimal_manifest(); + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + create_test_hash(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), create_test_hash()), + network: "mainnet".parse().expect("valid network"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.convert(&manifest); + + //* Then + assert!(result.is_err()); + let err = result.expect_err("should fail"); + assert!( + matches!(err, BridgeError::ReadIpcSchemaFile { .. }), + "should be ReadIpcSchemaFile error, got: {:?}", + err + ); + } + + // ============================================================ + // LegacyBridge::convert tests - functions + // ============================================================ + + #[test] + fn convert_manifest_with_function_inlines_source() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + setup_function_files(&dir); + + let mut manifest = create_minimal_manifest(); + manifest.functions.insert( + "decode".parse().expect("valid func name"), + AuthoringFunctionDef { + input_types: vec![DataType(datafusion::arrow::datatypes::DataType::Binary)], + output_type: DataType(datafusion::arrow::datatypes::DataType::Utf8), + source: FileRef::new("functions/decode.js".to_string(), create_test_hash()), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.convert(&manifest); + + //* Then + let legacy = result.expect("conversion should succeed"); + assert_eq!(legacy.functions.len(), 1); + + let func = legacy + .functions + .get(&"decode".parse().unwrap()) + .expect("should have function"); + + assert!(func.source.source.contains("function decode")); + assert_eq!(func.source.filename, "decode.js"); + } + + #[test] + fn convert_manifest_preserves_function_types() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + setup_function_files(&dir); + + let mut manifest = create_minimal_manifest(); + manifest.functions.insert( + "decode".parse().expect("valid func name"), + AuthoringFunctionDef { + input_types: vec![ + DataType(datafusion::arrow::datatypes::DataType::Binary), + DataType(datafusion::arrow::datatypes::DataType::Int64), + ], + output_type: DataType(datafusion::arrow::datatypes::DataType::Utf8), + source: FileRef::new("functions/decode.js".to_string(), create_test_hash()), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.convert(&manifest); + + //* Then + let legacy = result.expect("conversion should succeed"); + let func = legacy + .functions + .get(&"decode".parse().unwrap()) + .expect("should have function"); + + assert_eq!(func.input_types.len(), 2); + assert_eq!( + *func.input_types[0].as_arrow(), + datafusion::arrow::datatypes::DataType::Binary + ); + assert_eq!( + *func.input_types[1].as_arrow(), + datafusion::arrow::datatypes::DataType::Int64 + ); + assert_eq!( + *func.output_type.as_arrow(), + datafusion::arrow::datatypes::DataType::Utf8 + ); + } + + #[test] + fn convert_manifest_fails_on_missing_function_file() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + // Don't create the function file + + let mut manifest = create_minimal_manifest(); + manifest.functions.insert( + "decode".parse().expect("valid func name"), + AuthoringFunctionDef { + input_types: vec![], + output_type: DataType(datafusion::arrow::datatypes::DataType::Utf8), + source: FileRef::new("functions/decode.js".to_string(), create_test_hash()), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.convert(&manifest); + + //* Then + assert!(result.is_err()); + let err = result.expect_err("should fail"); + assert!( + matches!(err, BridgeError::ReadFunctionFile { .. }), + "should be ReadFunctionFile error, got: {:?}", + err + ); + } + + // ============================================================ + // LegacyBridge::to_json tests + // ============================================================ + + #[test] + fn to_json_produces_valid_json() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + setup_test_files(&dir); + + let mut manifest = create_minimal_manifest(); + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + create_test_hash(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), create_test_hash()), + network: "mainnet".parse().expect("valid network"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.to_json(&manifest); + + //* Then + let json = result.expect("to_json should succeed"); + + // Verify it's valid JSON by parsing it + let parsed: serde_json::Value = serde_json::from_str(&json).expect("should be valid JSON"); + + // Verify structure + assert_eq!(parsed["kind"], "manifest"); + assert!(parsed["tables"]["transfers"].is_object()); + assert!(parsed["tables"]["transfers"]["input"]["sql"].is_string()); + } + + #[test] + fn to_json_includes_inline_sql() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + setup_test_files(&dir); + + let mut manifest = create_minimal_manifest(); + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + create_test_hash(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), create_test_hash()), + network: "mainnet".parse().expect("valid network"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.to_json(&manifest); + + //* Then + let json = result.expect("to_json should succeed"); + assert!( + json.contains("CREATE TABLE transfers"), + "JSON should contain inline SQL" + ); + } + + // ============================================================ + // Dependencies conversion tests + // ============================================================ + + #[test] + fn convert_manifest_with_dependencies() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + + let mut manifest = create_minimal_manifest(); + manifest.dependencies.insert( + "eth".parse().expect("valid alias"), + DepInfo { + hash: create_test_hash(), + namespace: "dep_ns".parse().expect("valid namespace"), + name: "eth_mainnet".parse().expect("valid name"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.convert(&manifest); + + //* Then + let legacy = result.expect("conversion should succeed"); + assert_eq!(legacy.dependencies.len(), 1); + assert!(legacy.dependencies.contains_key(&"eth".parse().unwrap())); + + // Verify the DepReference has the correct format + let dep_ref = legacy.dependencies.get(&"eth".parse().unwrap()).unwrap(); + assert_eq!(dep_ref.fqn().namespace().to_string(), "dep_ns"); + assert_eq!(dep_ref.fqn().name().to_string(), "eth_mainnet"); + } + + // ============================================================ + // derived.spec.json schema compliance tests + // ============================================================ + + #[test] + fn legacy_output_has_required_kind_field() { + //* Given + // As per derived.spec.json: "kind" is required and must be "manifest" + let dir = TempDir::new().expect("should create temp dir"); + let manifest = create_minimal_manifest(); + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.to_json(&manifest); + + //* Then + let json = result.expect("to_json should succeed"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); + assert_eq!( + parsed["kind"], "manifest", + "legacy output must have kind='manifest' per derived.spec.json" + ); + } + + #[test] + fn legacy_output_table_has_required_fields() { + //* Given + // As per derived.spec.json Table definition: + // - "input" (required): TableInput containing View with "sql" + // - "schema" (required): TableSchema containing "arrow" with "fields" + // - "network" (required): NetworkId string + let dir = TempDir::new().expect("should create temp dir"); + setup_test_files(&dir); + + let mut manifest = create_minimal_manifest(); + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + create_test_hash(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), create_test_hash()), + network: "mainnet".parse().expect("valid network"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.to_json(&manifest); + + //* Then + let json = result.expect("to_json should succeed"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); + + let table = &parsed["tables"]["transfers"]; + + // Validate Table required fields per spec + assert!( + table["input"].is_object(), + "table must have 'input' object per derived.spec.json" + ); + assert!( + table["input"]["sql"].is_string(), + "table.input must have 'sql' string (View format) per derived.spec.json" + ); + assert!( + table["schema"].is_object(), + "table must have 'schema' object per derived.spec.json" + ); + assert!( + table["schema"]["arrow"].is_object(), + "table.schema must have 'arrow' object per derived.spec.json" + ); + assert!( + table["schema"]["arrow"]["fields"].is_array(), + "table.schema.arrow must have 'fields' array per derived.spec.json" + ); + assert!( + table["network"].is_string(), + "table must have 'network' string per derived.spec.json" + ); + } + + #[test] + fn legacy_output_schema_field_has_required_properties() { + //* Given + // As per derived.spec.json Field definition: + // - "name" (required): string + // - "type" (required): DataType string + // - "nullable" (required): boolean + let dir = TempDir::new().expect("should create temp dir"); + setup_test_files(&dir); + + let mut manifest = create_minimal_manifest(); + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + create_test_hash(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), create_test_hash()), + network: "mainnet".parse().expect("valid network"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.to_json(&manifest); + + //* Then + let json = result.expect("to_json should succeed"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); + + let fields = &parsed["tables"]["transfers"]["schema"]["arrow"]["fields"]; + assert!(fields.is_array(), "fields must be an array"); + + for field in fields.as_array().expect("fields is array") { + assert!( + field["name"].is_string(), + "field must have 'name' string per derived.spec.json" + ); + assert!( + field["type"].is_string(), + "field must have 'type' string per derived.spec.json" + ); + assert!( + field["nullable"].is_boolean(), + "field must have 'nullable' boolean per derived.spec.json" + ); + } + } + + #[test] + fn legacy_output_function_has_required_fields() { + //* Given + // As per derived.spec.json Function definition: + // - "inputTypes" (required): array of DataType strings + // - "outputType" (required): DataType string + // - "source" (required): FunctionSource with "source" and "filename" + let dir = TempDir::new().expect("should create temp dir"); + setup_function_files(&dir); + + let mut manifest = create_minimal_manifest(); + manifest.functions.insert( + "decode".parse().expect("valid func name"), + AuthoringFunctionDef { + input_types: vec![DataType(datafusion::arrow::datatypes::DataType::Binary)], + output_type: DataType(datafusion::arrow::datatypes::DataType::Utf8), + source: FileRef::new("functions/decode.js".to_string(), create_test_hash()), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.to_json(&manifest); + + //* Then + let json = result.expect("to_json should succeed"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); + + let func = &parsed["functions"]["decode"]; + + // Validate Function required fields per spec + assert!( + func["inputTypes"].is_array(), + "function must have 'inputTypes' array per derived.spec.json" + ); + assert!( + func["outputType"].is_string(), + "function must have 'outputType' string per derived.spec.json" + ); + assert!( + func["source"].is_object(), + "function must have 'source' object per derived.spec.json" + ); + assert!( + func["source"]["source"].is_string(), + "function.source must have 'source' string per derived.spec.json" + ); + assert!( + func["source"]["filename"].is_string(), + "function.source must have 'filename' string per derived.spec.json" + ); + } + + #[test] + fn legacy_output_dependency_has_correct_format() { + //* Given + // As per derived.spec.json DepReference: + // Format: `namespace/name@version` or `namespace/name@hash` + // Pattern: ^[a-z0-9_]+/[a-z_][a-z0-9_]*@.+$ + let dir = TempDir::new().expect("should create temp dir"); + + let mut manifest = create_minimal_manifest(); + manifest.dependencies.insert( + "eth".parse().expect("valid alias"), + DepInfo { + hash: create_test_hash(), + namespace: "dep_ns".parse().expect("valid namespace"), + name: "eth_mainnet".parse().expect("valid name"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.to_json(&manifest); + + //* Then + let json = result.expect("to_json should succeed"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); + + let dep_value = &parsed["dependencies"]["eth"]; + assert!( + dep_value.is_string(), + "dependency must be a string (DepReference format)" + ); + + let dep_str = dep_value.as_str().expect("dependency is string"); + // Validate DepReference pattern: namespace/name@hash_or_version + let pattern = regex::Regex::new(r"^[a-z0-9_]+/[a-z_][a-z0-9_]*@.+$").expect("valid regex"); + assert!( + pattern.is_match(dep_str), + "dependency reference '{}' must match pattern '^[a-z0-9_]+/[a-z_][a-z0-9_]*@.+$' per derived.spec.json", + dep_str + ); + } + + #[test] + fn legacy_output_complete_manifest_structure() { + //* Given + // Complete manifest with tables, functions, and dependencies + let dir = TempDir::new().expect("should create temp dir"); + setup_test_files(&dir); + setup_function_files(&dir); + + let mut manifest = create_minimal_manifest(); + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + create_test_hash(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), create_test_hash()), + network: "mainnet".parse().expect("valid network"), + }, + ); + manifest.functions.insert( + "decode".parse().expect("valid func name"), + AuthoringFunctionDef { + input_types: vec![DataType(datafusion::arrow::datatypes::DataType::Binary)], + output_type: DataType(datafusion::arrow::datatypes::DataType::Utf8), + source: FileRef::new("functions/decode.js".to_string(), create_test_hash()), + }, + ); + manifest.dependencies.insert( + "eth".parse().expect("valid alias"), + DepInfo { + hash: create_test_hash(), + namespace: "dep_ns".parse().expect("valid namespace"), + name: "eth_mainnet".parse().expect("valid name"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.to_json(&manifest); + + //* Then + let json = result.expect("to_json should succeed"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); + + // Verify top-level structure per derived.spec.json + assert_eq!(parsed["kind"], "manifest", "kind must be 'manifest'"); + assert!( + parsed["tables"].is_object(), + "tables must be present as object" + ); + assert!( + parsed["functions"].is_object(), + "functions must be present as object" + ); + assert!( + parsed["dependencies"].is_object(), + "dependencies must be present as object" + ); + + assert!( + parsed["start_block"].is_null(), + "start_block should be null" + ); + } +} diff --git a/crates/core/dataset-authoring/src/cache.rs b/crates/core/dataset-authoring/src/cache.rs new file mode 100644 index 000000000..256498345 --- /dev/null +++ b/crates/core/dataset-authoring/src/cache.rs @@ -0,0 +1,801 @@ +//! Global dependency cache. +//! +//! Provides caching for resolved manifests at `~/.amp/registry//`. +//! +//! The cache stores dependency packages locally to avoid repeated network +//! fetches during builds. Each package is stored in a directory named by +//! its content hash, containing: +//! +//! - `manifest.json` - Canonical manifest with file references +//! - `tables/
.sql` - SQL files for derived tables +//! - `tables/
.ipc` - Arrow IPC schema files +//! - `functions/.js` - Function source files + +use std::path::{Path, PathBuf}; + +use datafusion::arrow::datatypes::SchemaRef; +use datasets_common::{hash::Hash, table_name::TableName}; + +use crate::{arrow_ipc, dependency_manifest::DependencyManifest}; + +/// Default cache location relative to home directory. +const CACHE_DIR: &str = ".amp/registry"; + +/// Global cache for resolved dependency manifests. +/// +/// Stores manifests at `~/.amp/registry//manifest.json`. +/// The cache is content-addressed by manifest hash for deduplication. +#[derive(Debug, Clone)] +pub struct Cache { + /// Root directory for the cache. + root: PathBuf, +} + +/// Errors that occur during cache operations. +#[derive(Debug, thiserror::Error)] +pub enum CacheError { + /// Failed to determine home directory + /// + /// This occurs when the system cannot determine the user's home directory, + /// typically due to missing environment variables or unusual system configuration. + #[error("failed to determine home directory")] + NoHomeDir, + + /// Failed to create cache directory + /// + /// This occurs when the cache directory cannot be created, typically due to + /// permission issues or disk space constraints. + #[error("failed to create cache directory '{path}'")] + CreateDir { + path: PathBuf, + #[source] + source: std::io::Error, + }, + + /// Failed to read manifest from cache + /// + /// This occurs when reading an existing cached manifest fails, typically due to + /// file permissions or concurrent modification. + #[error("failed to read cached manifest for hash '{hash}'")] + ReadManifest { + hash: Hash, + #[source] + source: std::io::Error, + }, + + /// Failed to parse cached manifest JSON + /// + /// This occurs when the cached manifest file exists but contains invalid JSON + /// or does not match the expected manifest schema. + #[error("failed to parse cached manifest for hash '{hash}'")] + ParseManifest { + hash: Hash, + #[source] + source: serde_json::Error, + }, + + /// Failed to write manifest to cache + /// + /// This occurs when writing a manifest to the cache fails, typically due to + /// disk space constraints or permission issues. + #[error("failed to write manifest to cache for hash '{hash}'")] + WriteManifest { + hash: Hash, + #[source] + source: std::io::Error, + }, + + /// Failed to serialize manifest to JSON + /// + /// This occurs when converting the manifest to JSON fails, which indicates + /// an internal error in the manifest structure. + #[error("failed to serialize manifest for hash '{hash}'")] + SerializeManifest { + hash: Hash, + #[source] + source: serde_json::Error, + }, + + /// Failed to read SQL file from cached package. + #[error("failed to read SQL file for table '{table_name}'")] + ReadSqlFile { + table_name: TableName, + #[source] + source: std::io::Error, + }, + + /// Failed to read IPC schema file from cached package. + #[error("failed to read IPC schema file for table '{table_name}'")] + ReadIpcSchema { + table_name: TableName, + #[source] + source: arrow_ipc::IpcSchemaError, + }, + + /// Failed to read function source file from cached package. + #[error("failed to read function source file '{filename}'")] + ReadFunctionFile { + filename: String, + #[source] + source: std::io::Error, + }, +} + +impl Cache { + /// Creates a new cache with the default location. + /// + /// The default location is `~/.amp/registry/`. + /// + /// # Errors + /// + /// Returns [`CacheError::NoHomeDir`] if the home directory cannot be determined. + pub fn new() -> Result { + let home = dirs::home_dir().ok_or(CacheError::NoHomeDir)?; + Ok(Self { + root: home.join(CACHE_DIR), + }) + } + + /// Creates a cache at a custom location. + /// + /// Useful for testing or when a non-standard cache location is required. + pub fn with_root(root: PathBuf) -> Self { + Self { root } + } + + /// Returns the root directory of the cache. + pub fn root(&self) -> &PathBuf { + &self.root + } + + /// Returns the directory path for a given manifest hash. + /// + /// The path is `//` where hash is the full 64-character hex string. + pub fn dir_path(&self, hash: &Hash) -> PathBuf { + self.root.join(hash.as_str()) + } + + /// Returns the path to a manifest file for a given hash. + /// + /// The path is `//manifest.json`. + pub fn manifest_path(&self, hash: &Hash) -> PathBuf { + self.dir_path(hash).join("manifest.json") + } + + /// Checks if a manifest exists in the cache. + pub fn contains(&self, hash: &Hash) -> bool { + self.manifest_path(hash).exists() + } + + /// Gets a manifest from the cache if it exists. + /// + /// # Errors + /// + /// Returns an error if the manifest exists but cannot be read or parsed. + pub fn get(&self, hash: &Hash) -> Result, CacheError> { + let path = self.manifest_path(hash); + if !path.exists() { + return Ok(None); + } + + let contents = std::fs::read_to_string(&path).map_err(|err| CacheError::ReadManifest { + hash: hash.clone(), + source: err, + })?; + + let manifest = + serde_json::from_str(&contents).map_err(|err| CacheError::ParseManifest { + hash: hash.clone(), + source: err, + })?; + + Ok(Some(manifest)) + } + + /// Stores a manifest in the cache. + /// + /// Creates the cache directory if it doesn't exist. + /// + /// # Errors + /// + /// Returns an error if the directory cannot be created or the manifest + /// cannot be serialized or written. + pub fn put(&self, hash: &Hash, manifest: &DependencyManifest) -> Result<(), CacheError> { + let dir_path = self.dir_path(hash); + + // Create directory if needed + if !dir_path.exists() { + std::fs::create_dir_all(&dir_path).map_err(|err| CacheError::CreateDir { + path: dir_path.clone(), + source: err, + })?; + } + + // Serialize manifest + let contents = serde_json::to_string_pretty(manifest).map_err(|err| { + CacheError::SerializeManifest { + hash: hash.clone(), + source: err, + } + })?; + + // Write to file + let manifest_path = self.manifest_path(hash); + std::fs::write(&manifest_path, contents).map_err(|err| CacheError::WriteManifest { + hash: hash.clone(), + source: err, + })?; + + Ok(()) + } + + /// Gets a cached package by hash, returning access to all package files. + /// + /// Unlike [`get`](Self::get) which returns only the manifest, this method + /// returns a [`CachedPackage`] that provides access to the full package + /// structure including SQL files, IPC schemas, and function sources. + /// + /// # Errors + /// + /// Returns an error if the package exists but cannot be read or parsed. + pub fn get_package(&self, hash: &Hash) -> Result, CacheError> { + let dir_path = self.dir_path(hash); + let manifest_path = self.manifest_path(hash); + + if !manifest_path.exists() { + return Ok(None); + } + + let contents = + std::fs::read_to_string(&manifest_path).map_err(|err| CacheError::ReadManifest { + hash: hash.clone(), + source: err, + })?; + + let manifest = + serde_json::from_str(&contents).map_err(|err| CacheError::ParseManifest { + hash: hash.clone(), + source: err, + })?; + + Ok(Some(CachedPackage { + root: dir_path, + manifest, + })) + } +} + +/// A cached package providing access to all package files. +/// +/// This represents a complete package stored in the cache, including: +/// - The dependency manifest (parsed from `manifest.json`) +/// - SQL files in `tables/
.sql` +/// - Arrow IPC schema files in `tables/
.ipc` +/// - Function sources in `functions/.js` +/// +/// Use this when you need to access the actual file contents beyond just +/// the manifest metadata. +#[derive(Debug, Clone)] +pub struct CachedPackage { + /// Root directory of the cached package. + root: PathBuf, + /// Parsed manifest from manifest.json. + manifest: DependencyManifest, +} + +impl CachedPackage { + /// Returns the root directory of the cached package. + pub fn root(&self) -> &Path { + &self.root + } + + /// Returns the dependency manifest. + pub fn manifest(&self) -> &DependencyManifest { + &self.manifest + } + + /// Consumes the cached package and returns the manifest. + pub fn into_manifest(self) -> DependencyManifest { + self.manifest + } + + /// Returns the path to the tables directory. + pub fn tables_dir(&self) -> PathBuf { + self.root.join("tables") + } + + /// Returns the path to the functions directory. + pub fn functions_dir(&self) -> PathBuf { + self.root.join("functions") + } + + /// Returns the path to a table's SQL file. + pub fn sql_path(&self, table_name: &TableName) -> PathBuf { + self.tables_dir().join(format!("{table_name}.sql")) + } + + /// Returns the path to a table's IPC schema file. + pub fn ipc_path(&self, table_name: &TableName) -> PathBuf { + self.tables_dir().join(format!("{table_name}.ipc")) + } + + /// Returns the path to a function's source file. + pub fn function_path(&self, filename: &str) -> PathBuf { + self.functions_dir().join(filename) + } + + /// Reads the SQL content for a table. + /// + /// # Errors + /// + /// Returns [`CacheError::ReadSqlFile`] if the file cannot be read. + pub fn read_sql(&self, table_name: &TableName) -> Result { + let path = self.sql_path(table_name); + std::fs::read_to_string(&path).map_err(|err| CacheError::ReadSqlFile { + table_name: table_name.clone(), + source: err, + }) + } + + /// Reads the Arrow schema from a table's IPC file. + /// + /// # Errors + /// + /// Returns [`CacheError::ReadIpcSchema`] if the file cannot be read or parsed. + pub fn read_schema(&self, table_name: &TableName) -> Result { + let path = self.ipc_path(table_name); + arrow_ipc::read_ipc_schema(&path).map_err(|err| CacheError::ReadIpcSchema { + table_name: table_name.clone(), + source: err, + }) + } + + /// Reads the source content for a function. + /// + /// # Errors + /// + /// Returns [`CacheError::ReadFunctionFile`] if the file cannot be read. + pub fn read_function(&self, filename: &str) -> Result { + let path = self.function_path(filename); + std::fs::read_to_string(&path).map_err(|err| CacheError::ReadFunctionFile { + filename: filename.to_string(), + source: err, + }) + } + + /// Checks if a table has an IPC schema file. + pub fn has_ipc_schema(&self, table_name: &TableName) -> bool { + self.ipc_path(table_name).exists() + } + + /// Checks if a table has a SQL file. + pub fn has_sql(&self, table_name: &TableName) -> bool { + self.sql_path(table_name).exists() + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use datasets_common::dataset_kind_str::DatasetKindStr; + + use super::*; + + fn create_test_manifest() -> DependencyManifest { + DependencyManifest { + kind: DatasetKindStr::new("manifest".to_string()), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + } + } + + #[test] + fn cache_with_root_creates_cache_at_custom_path() { + //* Given + let root = PathBuf::from("/tmp/test-cache"); + + //* When + let cache = Cache::with_root(root.clone()); + + //* Then + assert_eq!(cache.root(), &root); + } + + #[test] + fn dir_path_returns_hash_directory() { + //* Given + let cache = Cache::with_root(PathBuf::from("/cache")); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + //* When + let path = cache.dir_path(&hash); + + //* Then + assert_eq!( + path, + PathBuf::from( + "/cache/b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + ) + ); + } + + #[test] + fn manifest_path_returns_manifest_file_path() { + //* Given + let cache = Cache::with_root(PathBuf::from("/cache")); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + //* When + let path = cache.manifest_path(&hash); + + //* Then + assert_eq!( + path, + PathBuf::from( + "/cache/b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9/manifest.json" + ) + ); + } + + #[test] + fn contains_returns_false_for_nonexistent_hash() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + //* When + let result = cache.contains(&hash); + + //* Then + assert!(!result, "cache should not contain nonexistent hash"); + } + + #[test] + fn get_returns_none_for_nonexistent_hash() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + //* When + let result = cache.get(&hash); + + //* Then + assert!(result.is_ok(), "get should succeed"); + assert!( + result.expect("should be ok").is_none(), + "should return None for nonexistent hash" + ); + } + + #[test] + fn put_and_get_roundtrip_succeeds() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let manifest = create_test_manifest(); + + //* When + let put_result = cache.put(&hash, &manifest); + + //* Then + assert!(put_result.is_ok(), "put should succeed"); + assert!(cache.contains(&hash), "cache should contain hash after put"); + + let get_result = cache.get(&hash); + assert!(get_result.is_ok(), "get should succeed"); + let retrieved = get_result + .expect("should be ok") + .expect("should have manifest"); + + // Verify manifest fields match + assert_eq!(retrieved.kind.as_str(), "manifest"); + assert!(retrieved.dependencies.is_empty()); + assert!(retrieved.tables.is_empty()); + } + + #[test] + fn put_creates_directory_if_needed() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let manifest = create_test_manifest(); + + // Verify directory doesn't exist + assert!(!cache.dir_path(&hash).exists()); + + //* When + let result = cache.put(&hash, &manifest); + + //* Then + assert!(result.is_ok(), "put should succeed"); + assert!( + cache.dir_path(&hash).exists(), + "directory should be created" + ); + assert!( + cache.manifest_path(&hash).exists(), + "manifest file should exist" + ); + } + + #[test] + fn get_with_corrupt_json_returns_parse_error() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + // Create directory and write invalid JSON + let dir_path = cache.dir_path(&hash); + std::fs::create_dir_all(&dir_path).expect("should create dir"); + std::fs::write(cache.manifest_path(&hash), "not valid json").expect("should write"); + + //* When + let result = cache.get(&hash); + + //* Then + assert!(result.is_err(), "get should fail with corrupt JSON"); + let err = result.expect_err("should have error"); + assert!( + matches!(err, CacheError::ParseManifest { .. }), + "should be ParseManifest error" + ); + } + + // ============================================================ + // CachedPackage tests + // ============================================================ + + #[test] + fn get_package_returns_none_for_nonexistent_hash() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + //* When + let result = cache.get_package(&hash); + + //* Then + assert!(result.is_ok(), "get_package should succeed"); + assert!( + result.expect("should be ok").is_none(), + "should return None for nonexistent hash" + ); + } + + #[test] + fn get_package_returns_cached_package() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let manifest = create_test_manifest(); + cache.put(&hash, &manifest).expect("put should succeed"); + + //* When + let result = cache.get_package(&hash); + + //* Then + assert!(result.is_ok(), "get_package should succeed"); + let package = result.expect("should be ok").expect("should have package"); + assert_eq!(package.manifest().kind.as_str(), "manifest"); + assert_eq!(package.root(), cache.dir_path(&hash)); + } + + #[test] + fn cached_package_path_methods_return_correct_paths() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let manifest = create_test_manifest(); + cache.put(&hash, &manifest).expect("put should succeed"); + + let package = cache + .get_package(&hash) + .expect("should succeed") + .expect("should have package"); + + //* When/Then + let table_name: TableName = "transfers".parse().expect("valid table name"); + assert!( + package + .sql_path(&table_name) + .ends_with("tables/transfers.sql") + ); + assert!( + package + .ipc_path(&table_name) + .ends_with("tables/transfers.ipc") + ); + assert!( + package + .function_path("decode.js") + .ends_with("functions/decode.js") + ); + } + + #[test] + fn cached_package_read_sql_succeeds() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let manifest = create_test_manifest(); + cache.put(&hash, &manifest).expect("put should succeed"); + + // Create SQL file in cache + let tables_dir = cache.dir_path(&hash).join("tables"); + std::fs::create_dir_all(&tables_dir).expect("should create dir"); + std::fs::write(tables_dir.join("test_table.sql"), "SELECT * FROM source") + .expect("should write"); + + let package = cache + .get_package(&hash) + .expect("should succeed") + .expect("should have package"); + + //* When + let table_name: TableName = "test_table".parse().expect("valid table name"); + let result = package.read_sql(&table_name); + + //* Then + assert!(result.is_ok(), "read_sql should succeed"); + assert_eq!(result.expect("should have content"), "SELECT * FROM source"); + } + + #[test] + fn cached_package_read_schema_succeeds() { + use std::sync::Arc; + + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let manifest = create_test_manifest(); + cache.put(&hash, &manifest).expect("put should succeed"); + + // Create IPC schema file in cache + let tables_dir = cache.dir_path(&hash).join("tables"); + std::fs::create_dir_all(&tables_dir).expect("should create dir"); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt64, false), + Field::new("name", DataType::Utf8, true), + ])); + arrow_ipc::write_ipc_schema(&schema, tables_dir.join("test_table.ipc")) + .expect("should write"); + + let package = cache + .get_package(&hash) + .expect("should succeed") + .expect("should have package"); + + //* When + let table_name: TableName = "test_table".parse().expect("valid table name"); + let result = package.read_schema(&table_name); + + //* Then + assert!(result.is_ok(), "read_schema should succeed"); + let read_schema = result.expect("should have schema"); + assert_eq!(read_schema.fields().len(), 2); + assert_eq!(read_schema.field(0).name(), "id"); + assert_eq!(read_schema.field(1).name(), "name"); + } + + #[test] + fn cached_package_read_function_succeeds() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let manifest = create_test_manifest(); + cache.put(&hash, &manifest).expect("put should succeed"); + + // Create function file in cache + let functions_dir = cache.dir_path(&hash).join("functions"); + std::fs::create_dir_all(&functions_dir).expect("should create dir"); + std::fs::write(functions_dir.join("decode.js"), "function decode() {}") + .expect("should write"); + + let package = cache + .get_package(&hash) + .expect("should succeed") + .expect("should have package"); + + //* When + let result = package.read_function("decode.js"); + + //* Then + assert!(result.is_ok(), "read_function should succeed"); + assert_eq!(result.expect("should have content"), "function decode() {}"); + } + + #[test] + fn cached_package_has_methods_work() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let manifest = create_test_manifest(); + cache.put(&hash, &manifest).expect("put should succeed"); + + // Create only SQL file + let tables_dir = cache.dir_path(&hash).join("tables"); + std::fs::create_dir_all(&tables_dir).expect("should create dir"); + std::fs::write(tables_dir.join("test_table.sql"), "SELECT 1").expect("should write"); + + let package = cache + .get_package(&hash) + .expect("should succeed") + .expect("should have package"); + + //* When/Then + let table_name: TableName = "test_table".parse().expect("valid table name"); + let missing_table: TableName = "missing".parse().expect("valid table name"); + + assert!(package.has_sql(&table_name), "should have SQL"); + assert!(!package.has_ipc_schema(&table_name), "should not have IPC"); + assert!(!package.has_sql(&missing_table), "should not have SQL"); + } + + #[test] + fn cached_package_into_manifest_returns_manifest() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let manifest = create_test_manifest(); + cache.put(&hash, &manifest).expect("put should succeed"); + + let package = cache + .get_package(&hash) + .expect("should succeed") + .expect("should have package"); + + //* When + let manifest = package.into_manifest(); + + //* Then + assert_eq!(manifest.kind.as_str(), "manifest"); + } +} diff --git a/crates/core/dataset-authoring/src/canonical.rs b/crates/core/dataset-authoring/src/canonical.rs new file mode 100644 index 000000000..0becd0eea --- /dev/null +++ b/crates/core/dataset-authoring/src/canonical.rs @@ -0,0 +1,357 @@ +//! Canonical JSON support for deterministic hashing. +//! +//! Provides RFC 8785 (JCS) compliant canonical JSON encoding for deterministic +//! content-addressable hashing. This module uses `serde_json_canonicalizer` for +//! canonicalization, producing output with the following properties: +//! +//! - Objects with lexicographically sorted keys +//! - No whitespace between tokens +//! - Numbers serialized without unnecessary precision +//! - Floating point numbers supported per RFC 8785 +//! - Consistent Unicode handling +//! +//! # Example +//! +//! ``` +//! use dataset_authoring::canonical::{canonicalize, hash_canonical}; +//! use serde_json::json; +//! +//! let value = json!({"b": 1, "a": 2}); +//! let canonical = canonicalize(&value).unwrap(); +//! assert_eq!(canonical, r#"{"a":2,"b":1}"#); +//! +//! let hash = hash_canonical(&value).unwrap(); +//! // hash is a datasets_common::Hash +//! ``` + +use datasets_common::hash::Hash; + +/// Errors that occur during JSON canonicalization. +#[derive(Debug, thiserror::Error)] +pub enum CanonicalizeError { + /// Canonicalization failed. + /// + /// This can occur if the value contains types that cannot be serialized + /// to canonical JSON form. + #[error("failed to canonicalize JSON")] + Canonicalize(#[source] serde_json::Error), +} + +/// Canonicalizes a JSON value to a deterministic string form per RFC 8785 (JCS). +/// +/// The canonical form has: +/// - Object keys sorted lexicographically +/// - No whitespace between tokens +/// - Numbers serialized per RFC 8785 (no unnecessary precision) +/// - Consistent encoding +/// +/// # Errors +/// +/// Returns [`CanonicalizeError::Canonicalize`] if the value cannot be serialized. +pub fn canonicalize(value: &serde_json::Value) -> Result { + let bytes = serde_json_canonicalizer::to_vec(value).map_err(CanonicalizeError::Canonicalize)?; + // SAFETY: JCS output is valid UTF-8 JSON + Ok(String::from_utf8(bytes).expect("JCS output should be valid UTF-8")) +} + +/// Canonicalizes and hashes a JSON value. +/// +/// This function: +/// 1. Serializes the value to canonical JSON per RFC 8785 (JCS) +/// 2. Computes the SHA-256 hash of the canonical bytes +/// +/// This is the standard way to compute content-addressable identities for +/// JSON manifests in the dataset authoring workflow. +/// +/// # Errors +/// +/// Returns [`CanonicalizeError::Canonicalize`] if the value cannot be serialized. +pub fn hash_canonical(value: &serde_json::Value) -> Result { + let canonical = canonicalize(value)?; + Ok(datasets_common::hash::hash(canonical.as_bytes())) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn canonicalize_with_unsorted_keys_produces_sorted_output() { + //* Given + let value = json!({"z": 1, "a": 2, "m": 3}); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + assert_eq!(canonical, r#"{"a":2,"m":3,"z":1}"#); + } + + #[test] + fn canonicalize_with_nested_objects_sorts_all_levels() { + //* Given + let value = json!({ + "outer": {"z": 1, "a": 2}, + "another": 3 + }); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + assert_eq!(canonical, r#"{"another":3,"outer":{"a":2,"z":1}}"#); + } + + #[test] + fn canonicalize_with_arrays_preserves_order() { + //* Given + let value = json!({"arr": [3, 1, 2]}); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + assert_eq!(canonical, r#"{"arr":[3,1,2]}"#); + } + + #[test] + fn canonicalize_removes_whitespace() { + //* Given + let value = json!({ + "key": "value", + "number": 42 + }); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + assert!( + !canonical.contains(' '), + "canonical JSON should have no spaces" + ); + assert!( + !canonical.contains('\n'), + "canonical JSON should have no newlines" + ); + } + + #[test] + fn canonicalize_handles_empty_object() { + //* Given + let value = json!({}); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + assert_eq!(canonical, "{}"); + } + + #[test] + fn canonicalize_handles_empty_array() { + //* Given + let value = json!([]); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + assert_eq!(canonical, "[]"); + } + + #[test] + fn canonicalize_handles_null() { + //* Given + let value = json!(null); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + assert_eq!(canonical, "null"); + } + + #[test] + fn canonicalize_handles_boolean_values() { + //* Given + let value = json!({"t": true, "f": false}); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + assert_eq!(canonical, r#"{"f":false,"t":true}"#); + } + + #[test] + fn canonicalize_handles_integer_numbers() { + //* Given + let value = json!({"pos": 42, "neg": -17, "zero": 0}); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + assert_eq!(canonical, r#"{"neg":-17,"pos":42,"zero":0}"#); + } + + #[test] + fn canonicalize_handles_string_with_special_chars() { + //* Given + // Test with quotes and backslashes which require escaping + let value = json!({"text": "hello\"world", "path": "a\\b"}); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + // Quotes and backslashes must be escaped + assert!( + canonical.contains(r#"\"#), + "should contain escaped characters" + ); + } + + #[test] + fn canonicalize_handles_unicode_strings() { + //* Given + let value = json!({"emoji": "\u{1F600}", "text": "caf\u{00E9}"}); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + // Unicode characters should be preserved + assert!(canonical.contains('\u{1F600}') || canonical.contains("\\u")); + } + + #[test] + fn hash_canonical_produces_consistent_hash() { + //* Given + let value = json!({"name": "test", "version": "1.0.0"}); + + //* When + let hash1 = hash_canonical(&value); + let hash2 = hash_canonical(&value); + + //* Then + let h1 = hash1.expect("first hash should succeed"); + let h2 = hash2.expect("second hash should succeed"); + assert_eq!(h1, h2, "same input should produce same hash"); + } + + #[test] + fn hash_canonical_differs_for_different_values() { + //* Given + let value1 = json!({"a": 1}); + let value2 = json!({"a": 2}); + + //* When + let hash1 = hash_canonical(&value1); + let hash2 = hash_canonical(&value2); + + //* Then + let h1 = hash1.expect("first hash should succeed"); + let h2 = hash2.expect("second hash should succeed"); + assert_ne!(h1, h2, "different inputs should produce different hashes"); + } + + #[test] + fn hash_canonical_is_order_independent() { + //* Given + // Same logical content, different key order in source + let value1 = json!({"b": 2, "a": 1}); + let value2 = json!({"a": 1, "b": 2}); + + //* When + let hash1 = hash_canonical(&value1); + let hash2 = hash_canonical(&value2); + + //* Then + let h1 = hash1.expect("first hash should succeed"); + let h2 = hash2.expect("second hash should succeed"); + assert_eq!(h1, h2, "canonicalization should make key order irrelevant"); + } + + // RFC 8785 (JCS) behavior tests + + #[test] + fn canonicalize_handles_floating_point_numbers() { + //* Given + // RFC 8785 allows floating point numbers + let value = json!({"number": 1.5, "whole": 2.0}); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("floating point numbers should be accepted"); + // 1.5 stays as 1.5, 2.0 becomes 2 (no unnecessary precision per RFC 8785) + assert_eq!(canonical, r#"{"number":1.5,"whole":2}"#); + } + + #[test] + fn canonicalize_accepts_integer_numbers() { + //* Given + // Integer numbers (including large ones) should work + let value = json!({ + "numbers": [0, 1, -1, 1000000000, -999999999] + }); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("integer canonicalization should succeed"); + assert_eq!(canonical, r#"{"numbers":[0,1,-1,1000000000,-999999999]}"#); + } + + #[test] + fn canonicalize_sorts_keys_lexicographically() { + //* Given + // Keys should be sorted by byte value (lexicographic) + let value = json!({ + "z": 1, + "A": 2, + "a": 3, + "1": 4 + }); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + // ASCII sort order: digits < uppercase < lowercase + assert_eq!(canonical, r#"{"1":4,"A":2,"a":3,"z":1}"#); + } + + #[test] + fn canonicalize_handles_literals() { + //* Given + let value = json!({ + "literals": [null, true, false] + }); + + //* When + let result = canonicalize(&value); + + //* Then + let canonical = result.expect("canonicalization should succeed"); + assert_eq!(canonical, r#"{"literals":[null,true,false]}"#); + } +} diff --git a/crates/core/dataset-authoring/src/config.rs b/crates/core/dataset-authoring/src/config.rs new file mode 100644 index 000000000..85571c681 --- /dev/null +++ b/crates/core/dataset-authoring/src/config.rs @@ -0,0 +1,1115 @@ +//! amp.yaml configuration parsing. +//! +//! Defines the [`AmpYaml`] struct for parsing and validating the `amp.yaml` authoring +//! configuration file. This is the entrypoint for dataset authoring. +//! +//! # amp.yaml Structure +//! +//! ```yaml +//! amp: 1.0.0 +//! namespace: my_namespace +//! name: my_dataset +//! version: 1.0.0 +//! dependencies: +//! eth: my_namespace/eth_mainnet@1.0.0 +//! tables: tables # optional; defaults to "tables" +//! functions: # optional +//! myFunc: +//! input_types: [Int64, Utf8] +//! output_type: Utf8 +//! source: functions/my_func.js +//! vars: # optional +//! my_var: default_value +//! ``` + +use std::{ + collections::BTreeMap, + fs, + path::{Component, Path, PathBuf}, +}; + +use datasets_common::{name::Name, namespace::Namespace, version::Version}; +use datasets_derived::{ + deps::{DepAlias, DepReference}, + func_name::FuncName, +}; +use semver::Version as SemverVersion; + +const SUPPORTED_AMP_MAJOR: u64 = 1; +const AMP_YAML_FILE: &str = "amp.yaml"; +const AMP_YML_FILE: &str = "amp.yml"; + +/// Authoring API contract version declared in `amp.yaml`. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, serde::Deserialize, +)] +#[serde(transparent)] +pub struct AmpVersion(SemverVersion); + +impl AmpVersion { + fn major(&self) -> u64 { + self.0.major + } +} + +impl std::fmt::Display for AmpVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl std::str::FromStr for AmpVersion { + type Err = semver::Error; + + fn from_str(s: &str) -> Result { + SemverVersion::parse(s).map(Self) + } +} + +/// User-defined function definition in the authoring config. +/// +/// Specifies the function signature and source file location. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FunctionDef { + /// Arrow data types for function input parameters. + pub input_types: Vec, + /// Arrow data type for function return value. + pub output_type: String, + /// Path to the function source file (relative to amp.yaml). + pub source: PathBuf, +} + +/// The amp.yaml configuration for dataset authoring. +/// +/// This struct represents the parsed and validated amp.yaml file, +/// which is the entrypoint for the dataset authoring workflow. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AmpYaml { + /// Authoring API contract version. + pub amp: AmpVersion, + /// Dataset namespace. + pub namespace: Namespace, + /// Dataset name. + pub name: Name, + /// Dataset version (semver). + pub version: Version, + /// External dataset dependencies with version requirements. + /// + /// Maps dependency aliases to their references (namespace/name@version or hash). + #[serde(default)] + pub dependencies: BTreeMap, + /// Directory containing table SQL files (`**/*.sql`). + /// + /// Each `.sql` file in this directory (recursively) defines a table. + /// The table name is derived from the file stem (must be unique across all files). + /// Defaults to `tables` when omitted. + #[serde(default = "default_tables_dir")] + pub tables: PathBuf, + /// Optional user-defined function definitions. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub functions: Option>, + /// Optional variables with default values. + /// + /// These can be overridden via CLI `--var` arguments. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vars: Option>, +} + +/// Versioned amp.yaml configuration (v1 schema). +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct AmpYamlV1 { + /// Dataset namespace. + pub namespace: Namespace, + /// Dataset name. + pub name: Name, + /// Dataset version (semver). + pub version: Version, + /// External dataset dependencies with version requirements. + /// + /// Maps dependency aliases to their references (namespace/name@version or hash). + #[serde(default)] + pub dependencies: BTreeMap, + /// Directory containing table SQL files (`**/*.sql`). + /// + /// Each `.sql` file in this directory (recursively) defines a table. + /// The table name is derived from the file stem (must be unique across all files). + /// Defaults to `tables` when omitted. + #[serde(default = "default_tables_dir")] + pub tables: PathBuf, + /// Optional user-defined function definitions. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub functions: Option>, + /// Optional variables with default values. + /// + /// These can be overridden via CLI `--var` arguments. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vars: Option>, +} + +enum AmpYamlVersioned { + V1(AmpYamlV1), +} + +fn default_tables_dir() -> PathBuf { + PathBuf::from("tables") +} + +impl AmpYaml { + /// Parse an `AmpYaml` configuration from a YAML string. + /// + /// # Errors + /// + /// Returns [`ConfigError::Parse`] if the YAML is invalid or doesn't match + /// the expected schema. + /// + /// Returns [`ConfigError::MissingAmpVersion`], [`ConfigError::InvalidAmpVersion`], + /// or [`ConfigError::UnsupportedAmpVersion`] if the authoring contract + /// version is missing, invalid, or unsupported. + /// + /// Returns [`ConfigError::Validation`] if the configuration fails validation + /// (e.g., no tables defined). + pub fn from_yaml(yaml: &str) -> Result { + let mut raw: serde_yaml::Value = + serde_yaml::from_str(yaml).map_err(|err| ConfigError::Parse { source: err })?; + + let amp_version = parse_amp_version(&raw)?; + + let versioned = match amp_version.major() { + SUPPORTED_AMP_MAJOR => { + remove_amp_field(&mut raw)?; + let v1: AmpYamlV1 = serde_yaml::from_value(raw) + .map_err(|err| ConfigError::Parse { source: err })?; + AmpYamlVersioned::V1(v1) + } + _ => { + return Err(ConfigError::UnsupportedAmpVersion { + version: amp_version.to_string(), + supported_major: SUPPORTED_AMP_MAJOR, + }); + } + }; + + let config = Self::from_versioned(amp_version, versioned)?; + config.validate()?; + Ok(config) + } + + /// Validate the configuration. + /// + /// # Errors + /// + /// Returns [`ConfigError::Validation`] if the configuration is invalid. + /// Returns [`ConfigError::InvalidPath`] if any path is absolute or uses `..` traversal. + fn validate(&self) -> Result<(), ConfigError> { + // Validate tables directory path + validate_path(&self.tables, "tables directory")?; + + // Validate function source paths + if let Some(functions) = &self.functions { + for (func_name, func_def) in functions { + validate_path(&func_def.source, &format!("function '{func_name}'"))?; + } + } + + Ok(()) + } + + fn from_versioned(amp: AmpVersion, versioned: AmpYamlVersioned) -> Result { + match (amp.major(), versioned) { + (SUPPORTED_AMP_MAJOR, AmpYamlVersioned::V1(v1)) => Ok(Self { + amp, + namespace: v1.namespace, + name: v1.name, + version: v1.version, + dependencies: v1.dependencies, + tables: v1.tables, + functions: v1.functions, + vars: v1.vars, + }), + _ => Err(ConfigError::UnsupportedAmpVersion { + version: amp.to_string(), + supported_major: SUPPORTED_AMP_MAJOR, + }), + } + } +} + +fn parse_amp_version(raw: &serde_yaml::Value) -> Result { + let mapping = raw + .as_mapping() + .ok_or_else(|| ConfigError::InvalidEnvelope { + message: "amp.yaml must be a mapping".to_string(), + })?; + let amp_value = mapping + .get(serde_yaml::Value::String("amp".to_string())) + .ok_or(ConfigError::MissingAmpVersion)?; + let amp_str = amp_value + .as_str() + .ok_or_else(|| ConfigError::InvalidAmpVersionType { + found: yaml_value_type(amp_value), + })?; + amp_str + .parse() + .map_err(|source| ConfigError::InvalidAmpVersion { + value: amp_str.to_string(), + source, + }) +} + +fn remove_amp_field(raw: &mut serde_yaml::Value) -> Result<(), ConfigError> { + let mapping = raw + .as_mapping_mut() + .ok_or_else(|| ConfigError::InvalidEnvelope { + message: "amp.yaml must be a mapping".to_string(), + })?; + mapping.remove(serde_yaml::Value::String("amp".to_string())); + Ok(()) +} + +fn yaml_value_type(value: &serde_yaml::Value) -> &'static str { + match value { + serde_yaml::Value::Null => "null", + serde_yaml::Value::Bool(_) => "bool", + serde_yaml::Value::Number(_) => "number", + serde_yaml::Value::String(_) => "string", + serde_yaml::Value::Sequence(_) => "sequence", + serde_yaml::Value::Mapping(_) => "mapping", + serde_yaml::Value::Tagged(_) => "tagged", + } +} + +/// Errors that occur during amp.yaml configuration parsing and validation. +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + /// YAML parsing failed. + /// + /// This occurs when the YAML syntax is invalid or the structure + /// doesn't match the expected schema. + #[error("failed to parse amp.yaml")] + Parse { + /// The underlying YAML parsing error. + #[source] + source: serde_yaml::Error, + }, + + /// Required amp.yaml envelope data is missing. + #[error("missing required 'amp' version in amp.yaml")] + MissingAmpVersion, + + /// The amp version value is not a string. + #[error("invalid 'amp' version: expected semver string, got {found}")] + InvalidAmpVersionType { + /// The YAML value type that was provided. + found: &'static str, + }, + + /// The amp version string is not valid semver. + #[error("invalid 'amp' version '{value}': {source}")] + InvalidAmpVersion { + /// The invalid version string. + value: String, + /// The underlying semver parse error. + #[source] + source: semver::Error, + }, + + /// The amp version major is not supported. + #[error("unsupported 'amp' version '{version}'; supported major versions: {supported_major}")] + UnsupportedAmpVersion { + /// The provided amp version. + version: String, + /// The supported major version. + supported_major: u64, + }, + + /// The amp.yaml envelope is malformed. + #[error("invalid amp.yaml: {message}")] + InvalidEnvelope { + /// Description of the envelope error. + message: String, + }, + + /// Configuration validation failed. + /// + /// This occurs when the parsed configuration doesn't meet + /// the required constraints (e.g., no tables defined). + #[error("invalid amp.yaml configuration: {message}")] + Validation { + /// Description of the validation failure. + message: String, + }, + + /// Invalid path in configuration. + /// + /// This occurs when a path in the configuration is absolute, + /// uses parent directory traversal (`..`), or is otherwise invalid. + #[error("invalid path '{path}': {reason}")] + InvalidPath { + /// The invalid path. + path: PathBuf, + /// Reason why the path is invalid. + reason: String, + }, +} + +/// Errors that occur while locating or loading amp.yaml/amp.yml configuration files. +#[derive(Debug, thiserror::Error)] +pub enum ConfigLoadError { + /// Both amp.yaml and amp.yml were found, creating ambiguity. + #[error( + "both amp.yaml and amp.yml found in {}; remove one to continue", + dir.display() + )] + Ambiguous { + /// Directory containing the configs. + dir: PathBuf, + /// Path to amp.yaml. + yaml: PathBuf, + /// Path to amp.yml. + yml: PathBuf, + }, + + /// No amp.yaml or amp.yml was found. + #[error( + "no amp.yaml or amp.yml found in {}; expected one configuration file", + dir.display() + )] + Missing { + /// Directory searched for the config. + dir: PathBuf, + }, + + /// Failed to read the config file. + #[error("failed to read config at {}", path.display())] + Read { + /// Path to the config file. + path: PathBuf, + /// The underlying I/O error. + #[source] + source: std::io::Error, + }, + + /// Failed to parse the config file. + #[error("failed to parse config at {}", path.display())] + Parse { + /// Path to the config file. + path: PathBuf, + /// The underlying config error. + #[source] + source: ConfigError, + }, +} + +/// Validate that a path is relative and doesn't use parent directory traversal. +/// +/// # Arguments +/// +/// * `path` - The path to validate. +/// * `context` - Description of where this path is used (for error messages). +/// +/// # Errors +/// +/// Returns [`ConfigError::InvalidPath`] if: +/// - The path is absolute +/// - The path contains parent directory traversal (`..`) +fn validate_path(path: &Path, context: &str) -> Result<(), ConfigError> { + // Check for absolute paths + if path.is_absolute() { + return Err(ConfigError::InvalidPath { + path: path.to_path_buf(), + reason: format!("{context} path must be relative"), + }); + } + + // Check for Windows drive letters (e.g., C:\path) which may not register as absolute on Unix + if let Some(s) = path.to_str() { + let mut chars = s.chars(); + if let (Some(first), Some(second)) = (chars.next(), chars.next()) + && first.is_ascii_alphabetic() + && second == ':' + { + return Err(ConfigError::InvalidPath { + path: path.to_path_buf(), + reason: format!("{context} path must be relative"), + }); + } + } + + // Check for parent directory traversal + for component in path.components() { + if component == Component::ParentDir { + return Err(ConfigError::InvalidPath { + path: path.to_path_buf(), + reason: format!("{context} path must not use parent directory traversal (..)"), + }); + } + } + + Ok(()) +} + +/// Locate the authoring config file (`amp.yaml` or `amp.yml`) in a directory. +/// +/// # Errors +/// +/// Returns [`ConfigLoadError::Ambiguous`] if both files exist. +/// Returns [`ConfigLoadError::Missing`] if neither file exists. +pub fn find_config_path(dir: &Path) -> Result { + let yaml_path = dir.join(AMP_YAML_FILE); + let yml_path = dir.join(AMP_YML_FILE); + let yaml_exists = yaml_path.is_file(); + let yml_exists = yml_path.is_file(); + + match (yaml_exists, yml_exists) { + (true, true) => Err(ConfigLoadError::Ambiguous { + dir: dir.to_path_buf(), + yaml: yaml_path, + yml: yml_path, + }), + (true, false) => Ok(yaml_path), + (false, true) => Ok(yml_path), + (false, false) => Err(ConfigLoadError::Missing { + dir: dir.to_path_buf(), + }), + } +} + +/// Load and parse the authoring config from a directory. +/// +/// Returns the resolved config path and parsed [`AmpYaml`] configuration. +/// +/// # Errors +/// +/// Returns [`ConfigLoadError`] if the file cannot be located, read, or parsed. +pub fn load_from_dir(dir: &Path) -> Result<(PathBuf, AmpYaml), ConfigLoadError> { + let path = find_config_path(dir)?; + let contents = fs::read_to_string(&path).map_err(|source| ConfigLoadError::Read { + path: path.clone(), + source, + })?; + let config = AmpYaml::from_yaml(&contents).map_err(|source| ConfigLoadError::Parse { + path: path.clone(), + source, + })?; + Ok((path, config)) +} + +#[cfg(test)] +mod tests { + use std::{fs, path::Path}; + + use super::*; + + #[test] + fn parse_minimal_valid_config() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let config = result.expect("should parse valid config"); + assert_eq!(config.amp.to_string(), "1.0.0"); + assert_eq!(config.namespace, "my_namespace"); + assert_eq!(config.name, "my_dataset"); + assert_eq!(config.version.to_string(), "1.0.0"); + assert!(config.dependencies.is_empty()); + assert_eq!(config.tables, PathBuf::from("tables")); + assert!(config.functions.is_none()); + assert!(config.vars.is_none()); + } + + #[test] + fn parse_full_config_with_all_fields() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.2.3 +dependencies: + eth: my_namespace/eth_mainnet@1.0.0 + other: other_ns/other_dataset@0.5.0 +tables: custom/models +functions: + myFunc: + input_types: [Int64, Utf8] + output_type: Utf8 + source: functions/my_func.js +vars: + network: mainnet + api_key: default_key +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let config = result.expect("should parse valid config"); + assert_eq!(config.amp.to_string(), "1.0.0"); + assert_eq!(config.namespace, "my_namespace"); + assert_eq!(config.name, "my_dataset"); + assert_eq!(config.version.to_string(), "1.2.3"); + assert_eq!(config.dependencies.len(), 2); + assert_eq!(config.tables, PathBuf::from("custom/models")); + assert!(config.functions.is_some()); + let funcs = config.functions.as_ref().expect("should have functions"); + assert_eq!(funcs.len(), 1); + assert!(config.vars.is_some()); + let vars = config.vars.as_ref().expect("should have vars"); + assert_eq!(vars.len(), 2); + assert_eq!(vars.get("network"), Some(&"mainnet".to_string())); + } + + #[test] + fn reject_config_without_amp_version() { + //* Given + let yaml = r#" +namespace: my_namespace +name: my_dataset +version: 1.0.0 +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject config without amp version"); + assert!(matches!(err, ConfigError::MissingAmpVersion)); + } + + #[test] + fn reject_config_with_invalid_amp_version() { + //* Given + let yaml = r#" +amp: not-a-version +namespace: my_namespace +name: my_dataset +version: 1.0.0 +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject invalid amp version"); + assert!(matches!(err, ConfigError::InvalidAmpVersion { .. })); + } + + #[test] + fn reject_config_with_start_block() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +start_block: 12345678 +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + assert!(result.is_err(), "start_block is no longer supported"); + } + + #[test] + fn reject_config_with_non_string_amp_version() { + //* Given + let yaml = r#" +amp: 123 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject non-string amp version"); + assert!(matches!(err, ConfigError::InvalidAmpVersionType { .. })); + } + + #[test] + fn reject_config_with_unsupported_amp_major() { + //* Given + let yaml = r#" +amp: 2.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject unsupported amp major"); + assert!(matches!(err, ConfigError::UnsupportedAmpVersion { .. })); + } + + #[test] + fn parse_config_with_hash_dependency() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +dependencies: + eth: my_namespace/eth_mainnet@b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let config = result.expect("should parse config with hash dependency"); + assert_eq!(config.dependencies.len(), 1); + } + + #[test] + fn default_tables_when_missing() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let config = result.expect("should default tables field"); + assert_eq!(config.tables, PathBuf::from("tables")); + } + + #[test] + fn reject_config_with_unknown_fields() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +unknown_field: some_value +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject config with unknown fields"); + assert!(matches!(err, ConfigError::Parse { .. })); + } + + #[test] + fn reject_config_with_invalid_namespace() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: Invalid-Namespace +name: my_dataset +version: 1.0.0 +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject config with invalid namespace"); + assert!(matches!(err, ConfigError::Parse { .. })); + } + + #[test] + fn reject_config_with_invalid_name() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: Invalid-Name +version: 1.0.0 +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject config with invalid name"); + assert!(matches!(err, ConfigError::Parse { .. })); + } + + #[test] + fn reject_config_with_invalid_version() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: not-a-version +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject config with invalid version"); + assert!(matches!(err, ConfigError::Parse { .. })); + } + + #[test] + fn reject_config_with_kind_field() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +kind: derived +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject config with kind field"); + let source = match err { + ConfigError::Parse { source } => source, + other => panic!("expected parse error, got: {other:?}"), + }; + let message = source.to_string(); + assert!( + message.contains("unknown field `kind`"), + "error should mention unknown 'kind' field: {message}" + ); + } + + #[test] + fn reject_config_with_invalid_dep_alias() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +dependencies: + 123invalid: my_namespace/eth_mainnet@1.0.0 +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject config with invalid dep alias"); + assert!(matches!(err, ConfigError::Parse { .. })); + } + + #[test] + fn reject_config_with_symbolic_dep_reference() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +dependencies: + eth: my_namespace/eth_mainnet@latest +tables: tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject symbolic dep reference like 'latest'"); + assert!(matches!(err, ConfigError::Parse { .. })); + } + + #[test] + fn reject_function_def_with_unknown_fields() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +tables: tables +functions: + myFunc: + input_types: [Int64] + output_type: Utf8 + source: functions/my_func.js + unknown_field: value +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject function with unknown fields"); + assert!(matches!(err, ConfigError::Parse { .. })); + } + + #[test] + fn validate_relative_path_succeeds() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +tables: custom/models +functions: + helper: + input_types: [Int64] + output_type: Utf8 + source: lib/funcs/helper.js +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + result.expect("should accept relative paths"); + } + + #[test] + fn validate_absolute_path_fails_for_tables() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +tables: /abs/path/tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject absolute path in tables"); + assert!( + matches!(err, ConfigError::InvalidPath { .. }), + "expected InvalidPath error, got: {err:?}" + ); + let message = err.to_string(); + assert!( + message.contains("must be relative"), + "error should mention 'must be relative': {message}" + ); + } + + #[test] + fn validate_absolute_path_fails_for_function() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +tables: tables +functions: + helper: + input_types: [Int64] + output_type: Utf8 + source: /abs/path/helper.js +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject absolute path in function"); + assert!( + matches!(err, ConfigError::InvalidPath { .. }), + "expected InvalidPath error, got: {err:?}" + ); + let message = err.to_string(); + assert!( + message.contains("must be relative"), + "error should mention 'must be relative': {message}" + ); + } + + #[test] + fn validate_windows_drive_path_fails() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +tables: "C:\\path\\tables" +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject Windows drive path"); + assert!( + matches!(err, ConfigError::InvalidPath { .. }), + "expected InvalidPath error, got: {err:?}" + ); + } + + #[test] + fn validate_parent_traversal_fails_for_tables() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +tables: sql/../tables +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject parent traversal in tables path"); + assert!( + matches!(err, ConfigError::InvalidPath { .. }), + "expected InvalidPath error, got: {err:?}" + ); + let message = err.to_string(); + assert!( + message.contains(".."), + "error should mention '..': {message}" + ); + } + + #[test] + fn validate_parent_traversal_fails_for_function() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +tables: tables +functions: + helper: + input_types: [Int64] + output_type: Utf8 + source: ../outside/helper.js +"#; + + //* When + let result = AmpYaml::from_yaml(yaml); + + //* Then + let err = result.expect_err("should reject parent traversal in function path"); + assert!( + matches!(err, ConfigError::InvalidPath { .. }), + "expected InvalidPath error, got: {err:?}" + ); + let message = err.to_string(); + assert!( + message.contains(".."), + "error should mention '..': {message}" + ); + } + + fn write_minimal_config(path: &Path) { + let yaml = r#" +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +"#; + fs::write(path, yaml).expect("should write config file"); + } + + #[test] + fn load_from_dir_prefers_amp_yaml_when_present() { + //* Given + let temp = tempfile::tempdir().expect("should create temp dir"); + let yaml_path = temp.path().join("amp.yaml"); + write_minimal_config(&yaml_path); + + //* When + let (path, config) = load_from_dir(temp.path()).expect("should load amp.yaml"); + + //* Then + assert_eq!(path, yaml_path); + assert_eq!(config.name, "my_dataset"); + } + + #[test] + fn load_from_dir_supports_amp_yml() { + //* Given + let temp = tempfile::tempdir().expect("should create temp dir"); + let yml_path = temp.path().join("amp.yml"); + write_minimal_config(&yml_path); + + //* When + let (path, config) = load_from_dir(temp.path()).expect("should load amp.yml"); + + //* Then + assert_eq!(path, yml_path); + assert_eq!(config.namespace, "my_namespace"); + } + + #[test] + fn load_from_dir_rejects_ambiguous_configs() { + //* Given + let temp = tempfile::tempdir().expect("should create temp dir"); + let yaml_path = temp.path().join("amp.yaml"); + let yml_path = temp.path().join("amp.yml"); + write_minimal_config(&yaml_path); + write_minimal_config(&yml_path); + + //* When + let err = load_from_dir(temp.path()).expect_err("should reject ambiguity"); + + //* Then + assert!( + matches!(err, ConfigLoadError::Ambiguous { .. }), + "expected ambiguous error, got: {err:?}" + ); + } + + #[test] + fn load_from_dir_errors_when_missing_config() { + //* Given + let temp = tempfile::tempdir().expect("should create temp dir"); + + //* When + let err = load_from_dir(temp.path()).expect_err("should reject missing config"); + + //* Then + assert!( + matches!(err, ConfigLoadError::Missing { .. }), + "expected missing error, got: {err:?}" + ); + } +} diff --git a/crates/core/dataset-authoring/src/dependency_manifest.rs b/crates/core/dataset-authoring/src/dependency_manifest.rs new file mode 100644 index 000000000..489b7bbce --- /dev/null +++ b/crates/core/dataset-authoring/src/dependency_manifest.rs @@ -0,0 +1,51 @@ +//! Dependency manifest types for dataset authoring. +//! +//! These types capture the minimal manifest shape needed for dependency +//! resolution and schema inference across derived and raw dataset kinds. + +use std::collections::BTreeMap; + +use datasets_common::{ + dataset_kind_str::DatasetKindStr, + manifest::{Function, TableSchema}, + network_id::NetworkId, + table_name::TableName, +}; +use datasets_derived::{ + deps::{DepAlias, DepReference}, + func_name::FuncName, +}; + +/// Minimal manifest representation for dependency resolution. +/// +/// This intentionally captures only the fields required for planning: +/// - `kind` to identify dataset type +/// - `dependencies` for transitive resolution +/// - `tables` for schema inference and network inference +/// - `functions` for validating dependency UDFs +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct DependencyManifest { + /// Dataset kind (e.g., "manifest", "evm-rpc", "firehose", "solana"). + pub kind: DatasetKindStr, + + /// External dataset dependencies with version/hash requirements. + #[serde(default)] + pub dependencies: BTreeMap, + + /// Table definitions mapped by table name. + #[serde(default)] + pub tables: BTreeMap, + + /// Function definitions mapped by function name. + #[serde(default)] + pub functions: BTreeMap, +} + +/// Minimal table definition for dependency planning. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct DependencyTable { + /// Arrow schema for the table. + pub schema: TableSchema, + /// Network for the table. + pub network: NetworkId, +} diff --git a/crates/core/dataset-authoring/src/discovery.rs b/crates/core/dataset-authoring/src/discovery.rs new file mode 100644 index 000000000..8a06080f2 --- /dev/null +++ b/crates/core/dataset-authoring/src/discovery.rs @@ -0,0 +1,344 @@ +//! Table file discovery for dbt-style dataset authoring. +//! +//! This module provides functionality to scan a tables directory and discover +//! all SQL table files, deriving table names from file stems. +//! +//! # Table Discovery Rules +//! +//! - Scans the tables directory recursively for `**/*.sql` files +//! - Table name is derived from the file stem (e.g., `transfers.sql` → `transfers`) +//! - Table names must be valid [`TableName`]s (SQL identifier rules) +//! - Duplicate table names across subdirectories are rejected +//! - Non-SQL files are ignored + +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, +}; + +use datasets_common::table_name::{TableName, TableNameError}; +use walkdir::WalkDir; + +/// Errors that can occur during table discovery. +#[derive(Debug, thiserror::Error)] +pub enum DiscoveryError { + /// Duplicate table name found in different paths. + #[error("duplicate table name '{name}': {path1} and {path2}")] + DuplicateTableName { + /// The conflicting table name. + name: String, + /// First path where the table was found. + path1: PathBuf, + /// Second path where the table was found. + path2: PathBuf, + }, + + /// Invalid table name derived from file stem. + #[error("invalid table name from file '{}'", path.display())] + InvalidTableName { + /// The file path with the invalid name. + path: PathBuf, + /// The underlying table name error. + #[source] + source: TableNameError, + }, + + /// I/O error reading directory. + #[error("failed to read tables directory")] + IoError(#[source] std::io::Error), + + /// Tables directory not found. + #[error("tables directory not found: {}", path.display())] + DirectoryNotFound { + /// The missing directory path. + path: PathBuf, + }, +} + +/// Discovered table information. +#[derive(Debug, Clone)] +pub struct DiscoveredTable { + /// The validated table name derived from the file stem. + pub name: TableName, + /// Path to the SQL file relative to the base directory. + pub path: PathBuf, +} + +/// Discover table SQL files in the given tables directory. +/// +/// Scans the tables directory recursively for `**/*.sql` files and returns +/// a map of table names to their file paths. The table name is derived from +/// the file stem (without extension). +/// +/// # Arguments +/// +/// * `base_dir` - The base directory containing the amp.yaml +/// * `tables_rel` - The tables directory path relative to base_dir +/// +/// # Errors +/// +/// Returns [`DiscoveryError`] if: +/// - The tables directory does not exist +/// - An I/O error occurs while reading the directory +/// - A file stem cannot be parsed as a valid [`TableName`] +/// - Duplicate table names are found across different subdirectories +/// +/// # Example +/// +/// ```ignore +/// use dataset_authoring::discovery::discover_tables; +/// use std::path::Path; +/// +/// let tables = discover_tables(Path::new("."), Path::new("tables"))?; +/// for (name, path) in &tables { +/// println!("Found table {} at {}", name, path.display()); +/// } +/// ``` +pub fn discover_tables( + base_dir: &Path, + tables_rel: &Path, +) -> Result, DiscoveryError> { + let tables_dir = base_dir.join(tables_rel); + + // Check if tables directory exists + if !tables_dir.is_dir() { + return Err(DiscoveryError::DirectoryNotFound { path: tables_dir }); + } + + let mut tables: BTreeMap = BTreeMap::new(); + + // Walk the directory recursively + for entry in WalkDir::new(&tables_dir) + .follow_links(false) + .into_iter() + .filter_map(|e| e.ok()) + { + let path = entry.path(); + + // Skip directories and non-SQL files + if !path.is_file() { + continue; + } + + let extension = path.extension().and_then(|e| e.to_str()); + if extension != Some("sql") { + continue; + } + + // Get the file stem as the table name + let file_stem = path + .file_stem() + .and_then(|s| s.to_str()) + .ok_or_else(|| DiscoveryError::IoError(std::io::Error::other("invalid file name")))?; + + // Parse as TableName to validate + let table_name: TableName = + file_stem + .parse() + .map_err(|source| DiscoveryError::InvalidTableName { + path: path.to_path_buf(), + source, + })?; + + // Compute relative path from base_dir (not tables_dir) + let relative_path = path + .strip_prefix(base_dir) + .map_err(|e| DiscoveryError::IoError(std::io::Error::other(e.to_string())))? + .to_path_buf(); + + // Check for duplicate names + if let Some(existing_path) = tables.get(&table_name) { + return Err(DiscoveryError::DuplicateTableName { + name: table_name.to_string(), + path1: existing_path.clone(), + path2: relative_path, + }); + } + + tables.insert(table_name, relative_path); + } + + Ok(tables) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + fn create_test_dir() -> TempDir { + tempfile::tempdir().expect("failed to create temp dir") + } + + fn create_file(dir: &Path, relative_path: &str, content: &str) { + let path = dir.join(relative_path); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("failed to create parent dirs"); + } + fs::write(&path, content).expect("failed to write file"); + } + + #[test] + fn discover_tables_finds_sql_files() { + //* Given + let temp_dir = create_test_dir(); + let base = temp_dir.path(); + create_file(base, "tables/transfers.sql", "SELECT 1"); + + //* When + let result = discover_tables(base, Path::new("tables")); + + //* Then + let tables = result.expect("should discover tables"); + assert_eq!(tables.len(), 1); + assert!(tables.contains_key(&"transfers".parse::().unwrap())); + } + + #[test] + fn discover_tables_recursive() { + //* Given + let temp_dir = create_test_dir(); + let base = temp_dir.path(); + create_file(base, "tables/transfers.sql", "SELECT 1"); + create_file(base, "tables/subdir/balances.sql", "SELECT 2"); + create_file(base, "tables/deep/nested/events.sql", "SELECT 3"); + + //* When + let result = discover_tables(base, Path::new("tables")); + + //* Then + let tables = result.expect("should discover tables recursively"); + assert_eq!(tables.len(), 3); + assert!(tables.contains_key(&"transfers".parse::().unwrap())); + assert!(tables.contains_key(&"balances".parse::().unwrap())); + assert!(tables.contains_key(&"events".parse::().unwrap())); + } + + #[test] + fn discover_tables_rejects_duplicate_names() { + //* Given + let temp_dir = create_test_dir(); + let base = temp_dir.path(); + create_file(base, "tables/transfers.sql", "SELECT 1"); + create_file(base, "tables/subdir/transfers.sql", "SELECT 2"); + + //* When + let result = discover_tables(base, Path::new("tables")); + + //* Then + let err = result.expect_err("should reject duplicate table names"); + assert!( + matches!(err, DiscoveryError::DuplicateTableName { name, .. } if name == "transfers") + ); + } + + #[test] + fn discover_tables_returns_sorted() { + //* Given + let temp_dir = create_test_dir(); + let base = temp_dir.path(); + create_file(base, "tables/zebra.sql", "SELECT 1"); + create_file(base, "tables/apple.sql", "SELECT 2"); + create_file(base, "tables/mango.sql", "SELECT 3"); + + //* When + let result = discover_tables(base, Path::new("tables")); + + //* Then + let tables = result.expect("should discover tables"); + let names: Vec<&str> = tables.keys().map(|n| n.as_str()).collect(); + assert_eq!(names, vec!["apple", "mango", "zebra"]); + } + + #[test] + fn discover_tables_rejects_invalid_names() { + //* Given + let temp_dir = create_test_dir(); + let base = temp_dir.path(); + create_file(base, "tables/123invalid.sql", "SELECT 1"); + + //* When + let result = discover_tables(base, Path::new("tables")); + + //* Then + let err = result.expect_err("should reject invalid table name"); + assert!(matches!(err, DiscoveryError::InvalidTableName { .. })); + } + + #[test] + fn discover_tables_ignores_non_sql_files() { + //* Given + let temp_dir = create_test_dir(); + let base = temp_dir.path(); + create_file(base, "tables/transfers.sql", "SELECT 1"); + create_file(base, "tables/readme.md", "# Documentation"); + create_file(base, "tables/notes.txt", "Some notes"); + create_file(base, "tables/config.json", "{}"); + + //* When + let result = discover_tables(base, Path::new("tables")); + + //* Then + let tables = result.expect("should discover only SQL files"); + assert_eq!(tables.len(), 1); + assert!(tables.contains_key(&"transfers".parse::().unwrap())); + } + + #[test] + fn discover_tables_fails_if_directory_not_found() { + //* Given + let temp_dir = create_test_dir(); + let base = temp_dir.path(); + // Don't create the tables directory + + //* When + let result = discover_tables(base, Path::new("tables")); + + //* Then + let err = result.expect_err("should fail if directory not found"); + assert!(matches!(err, DiscoveryError::DirectoryNotFound { .. })); + } + + #[test] + fn discover_tables_handles_empty_directory() { + //* Given + let temp_dir = create_test_dir(); + let base = temp_dir.path(); + fs::create_dir_all(base.join("tables")).expect("failed to create tables dir"); + + //* When + let result = discover_tables(base, Path::new("tables")); + + //* Then + let tables = result.expect("should handle empty directory"); + assert!(tables.is_empty()); + } + + #[test] + fn discover_tables_returns_relative_paths_from_base() { + //* Given + let temp_dir = create_test_dir(); + let base = temp_dir.path(); + create_file(base, "tables/transfers.sql", "SELECT 1"); + create_file(base, "tables/subdir/balances.sql", "SELECT 2"); + + //* When + let result = discover_tables(base, Path::new("tables")); + + //* Then + let tables = result.expect("should discover tables"); + let transfers_path = tables + .get(&"transfers".parse::().unwrap()) + .unwrap(); + let balances_path = tables + .get(&"balances".parse::().unwrap()) + .unwrap(); + + // Paths should be relative to base_dir (include tables/ prefix) + assert_eq!(transfers_path, &PathBuf::from("tables/transfers.sql")); + assert_eq!(balances_path, &PathBuf::from("tables/subdir/balances.sql")); + } +} diff --git a/crates/core/dataset-authoring/src/files.rs b/crates/core/dataset-authoring/src/files.rs new file mode 100644 index 000000000..75f49a012 --- /dev/null +++ b/crates/core/dataset-authoring/src/files.rs @@ -0,0 +1,619 @@ +//! File hashing and path normalization utilities. +//! +//! This module provides utilities for working with files in the dataset authoring +//! workflow: +//! +//! - **File hashing**: Compute SHA-256 hashes of file contents +//! - **Path normalization**: Convert paths to POSIX-style relative paths +//! - **File references**: Combine path + hash for manifest entries +//! +//! # Path Normalization +//! +//! Paths in manifests must follow these rules: +//! - Relative to the package root (no absolute paths) +//! - POSIX separators (`/`) only +//! - No `./` prefix +//! - No `..` components +//! +//! # Example +//! +//! ```no_run +//! use std::path::Path; +//! +//! use dataset_authoring::files::{FileRef, hash_file, normalize_path}; +//! +//! let base = Path::new("/project"); +//! let file = Path::new("/project/tables/users.sql"); +//! +//! // Compute file hash +//! let hash = hash_file(file).unwrap(); +//! +//! // Normalize path relative to base +//! let path = normalize_path(file, base).unwrap(); +//! assert_eq!(path, "tables/users.sql"); +//! +//! // Create file reference for manifest +//! let file_ref = FileRef { path, hash }; +//! ``` + +use std::{fs, io, path::Path}; + +use datasets_common::hash::Hash; +use serde::{Deserialize, Serialize}; + +/// A reference to a file with its normalized path and content hash. +/// +/// File references are used in manifests to identify specific file contents +/// by their content-addressable hash. The path is relative to the package +/// root using POSIX separators. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FileRef { + /// Relative path to the file using POSIX separators. + /// + /// Must not start with `./` or contain `..` components. + pub path: String, + + /// SHA-256 hash of the file contents. + pub hash: Hash, +} + +impl FileRef { + /// Creates a new file reference from a path and hash. + pub fn new(path: String, hash: Hash) -> Self { + Self { path, hash } + } + + /// Creates a file reference by reading and hashing a file. + /// + /// # Arguments + /// + /// * `file_path` - Absolute path to the file to hash + /// * `base_path` - Base directory for computing relative paths + /// + /// # Errors + /// + /// Returns [`FileRefError::HashFile`] if the file cannot be read. + /// Returns [`FileRefError::NormalizePath`] if the path cannot be normalized. + pub fn from_file(file_path: &Path, base_path: &Path) -> Result { + let hash = hash_file(file_path).map_err(|err| FileRefError::HashFile { + path: file_path.to_path_buf(), + source: err, + })?; + let path = normalize_path(file_path, base_path).map_err(FileRefError::NormalizePath)?; + Ok(Self { path, hash }) + } +} + +/// Errors that occur when creating a file reference. +#[derive(Debug, thiserror::Error)] +pub enum FileRefError { + /// Failed to read and hash file contents. + #[error("failed to hash file '{}'", path.display())] + HashFile { + /// Path to the file that failed to hash. + path: std::path::PathBuf, + /// Underlying I/O error. + #[source] + source: io::Error, + }, + + /// Failed to normalize the file path. + #[error("failed to normalize path")] + NormalizePath(#[source] PathError), +} + +/// Computes the SHA-256 hash of a file's contents. +/// +/// Reads the entire file into memory and computes its hash using the +/// `datasets_common::hash::hash` function. +/// +/// # Arguments +/// +/// * `path` - Path to the file to hash +/// +/// # Errors +/// +/// Returns an I/O error if the file cannot be read. +/// +/// # Example +/// +/// ```no_run +/// use std::path::Path; +/// +/// use dataset_authoring::files::hash_file; +/// +/// let hash = hash_file(Path::new("tables/users.sql")).unwrap(); +/// println!("File hash: {}", hash); +/// ``` +pub fn hash_file(path: &Path) -> Result { + let contents = fs::read(path)?; + Ok(datasets_common::hash::hash(&contents)) +} + +/// Errors that occur during path normalization. +#[derive(Debug, thiserror::Error)] +pub enum PathError { + /// Path is absolute when a relative path is required. + #[error("path must be relative, got absolute path '{}'", path.display())] + AbsolutePath { + /// The absolute path that was provided. + path: std::path::PathBuf, + }, + + /// Path is not within the base directory. + /// + /// This occurs when the path escapes the base directory via `..` components + /// or when the path is not a descendant of the base. + #[error("path '{}' is not within base directory '{}'", path.display(), base.display())] + NotWithinBase { + /// The path that is not within the base. + path: std::path::PathBuf, + /// The base directory. + base: std::path::PathBuf, + }, + + /// Path contains disallowed `..` component. + #[error("path contains disallowed '..' component: '{}'", path.display())] + ContainsParentRef { + /// The path containing `..`. + path: std::path::PathBuf, + }, + + /// Path starts with `./` which is not allowed. + #[error("path must not start with './': '{}'", path.display())] + StartsWithCurrentDir { + /// The path starting with `./`. + path: std::path::PathBuf, + }, + + /// Failed to strip the base prefix from the path. + #[error("failed to compute relative path from '{}' to '{}'", base.display(), path.display())] + StripPrefixFailed { + /// The path that could not be made relative. + path: std::path::PathBuf, + /// The base directory. + base: std::path::PathBuf, + }, +} + +/// Normalizes a file path to a POSIX-style relative path. +/// +/// This function converts a path to a normalized form suitable for manifests: +/// - Computes the path relative to the base directory +/// - Uses POSIX separators (`/`) regardless of platform +/// - Rejects paths that start with `./` +/// - Rejects paths containing `..` components +/// - Rejects absolute paths in the result +/// +/// # Arguments +/// +/// * `path` - The path to normalize (can be absolute or relative) +/// * `base` - The base directory for computing relative paths +/// +/// # Errors +/// +/// Returns [`PathError`] if the path cannot be normalized: +/// - [`PathError::NotWithinBase`] if the path is not under the base directory +/// - [`PathError::ContainsParentRef`] if the relative path contains `..` +/// - [`PathError::StartsWithCurrentDir`] if the relative path starts with `./` +/// - [`PathError::AbsolutePath`] if the result would be an absolute path +/// +/// # Example +/// +/// ``` +/// use std::path::Path; +/// +/// use dataset_authoring::files::normalize_path; +/// +/// let base = Path::new("/project"); +/// let file = Path::new("/project/tables/users.sql"); +/// +/// let normalized = normalize_path(file, base).unwrap(); +/// assert_eq!(normalized, "tables/users.sql"); +/// ``` +pub fn normalize_path(path: &Path, base: &Path) -> Result { + // Get canonical forms to handle symlinks and relative components + // We use dunce::canonicalize on Windows to avoid \\?\ prefixes, but + // for now we just use the paths as-is since we're dealing with logical paths + + // Strip the base prefix to get the relative path + let relative = path + .strip_prefix(base) + .map_err(|_| PathError::NotWithinBase { + path: path.to_path_buf(), + base: base.to_path_buf(), + })?; + + // Convert to string with POSIX separators + let mut normalized = String::new(); + let mut first = true; + + for component in relative.components() { + use std::path::Component; + + match component { + Component::Normal(s) => { + if !first { + normalized.push('/'); + } + // Convert OsStr to str, rejecting non-UTF8 paths + let s = s.to_str().ok_or_else(|| PathError::NotWithinBase { + path: path.to_path_buf(), + base: base.to_path_buf(), + })?; + normalized.push_str(s); + first = false; + } + Component::ParentDir => { + return Err(PathError::ContainsParentRef { + path: path.to_path_buf(), + }); + } + Component::CurDir => { + // Skip `.` components but check if it would result in `./` prefix + if first { + return Err(PathError::StartsWithCurrentDir { + path: path.to_path_buf(), + }); + } + } + Component::RootDir | Component::Prefix(_) => { + return Err(PathError::AbsolutePath { + path: path.to_path_buf(), + }); + } + } + } + + // Reject empty paths (path == base) + if normalized.is_empty() { + return Err(PathError::NotWithinBase { + path: path.to_path_buf(), + base: base.to_path_buf(), + }); + } + + Ok(normalized) +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use super::*; + + // ============================================================ + // hash_file tests + // ============================================================ + + #[test] + fn hash_file_with_known_content_produces_expected_hash() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let file_path = dir.path().join("test.txt"); + std::fs::write(&file_path, b"hello world").expect("should write file"); + + // SHA-256 of "hello world" + let expected = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"; + + //* When + let result = hash_file(&file_path); + + //* Then + let hash = result.expect("hashing should succeed"); + assert_eq!(hash.as_str(), expected); + } + + #[test] + fn hash_file_with_empty_file_produces_empty_hash() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let file_path = dir.path().join("empty.txt"); + std::fs::write(&file_path, b"").expect("should write file"); + + // SHA-256 of empty string + let expected = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + //* When + let result = hash_file(&file_path); + + //* Then + let hash = result.expect("hashing should succeed"); + assert_eq!(hash.as_str(), expected); + } + + #[test] + fn hash_file_with_nonexistent_file_returns_error() { + //* Given + let path = Path::new("/nonexistent/path/to/file.txt"); + + //* When + let result = hash_file(path); + + //* Then + assert!(result.is_err(), "should fail for nonexistent file"); + let err = result.expect_err("should return error"); + assert_eq!(err.kind(), io::ErrorKind::NotFound); + } + + #[test] + fn hash_file_is_deterministic() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let file_path = dir.path().join("test.txt"); + std::fs::write(&file_path, b"deterministic content").expect("should write file"); + + //* When + let hash1 = hash_file(&file_path).expect("first hash should succeed"); + let hash2 = hash_file(&file_path).expect("second hash should succeed"); + + //* Then + assert_eq!(hash1, hash2, "same file should produce same hash"); + } + + #[test] + fn hash_file_with_binary_content_succeeds() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let file_path = dir.path().join("binary.bin"); + let binary_content: Vec = (0..=255).collect(); + std::fs::write(&file_path, &binary_content).expect("should write file"); + + //* When + let result = hash_file(&file_path); + + //* Then + assert!(result.is_ok(), "should hash binary content"); + } + + // ============================================================ + // normalize_path tests + // ============================================================ + + #[test] + fn normalize_path_with_simple_relative_path_succeeds() { + //* Given + let base = Path::new("/project"); + let path = Path::new("/project/tables/users.sql"); + + //* When + let result = normalize_path(path, base); + + //* Then + let normalized = result.expect("normalization should succeed"); + assert_eq!(normalized, "tables/users.sql"); + } + + #[test] + fn normalize_path_with_nested_path_uses_posix_separators() { + //* Given + let base = Path::new("/project"); + let path = Path::new("/project/tables/nested/users.sql"); + + //* When + let result = normalize_path(path, base); + + //* Then + let normalized = result.expect("normalization should succeed"); + assert_eq!(normalized, "tables/nested/users.sql"); + assert!(!normalized.contains('\\'), "should not contain backslashes"); + } + + #[test] + fn normalize_path_with_single_file_succeeds() { + //* Given + let base = Path::new("/project"); + let path = Path::new("/project/manifest.json"); + + //* When + let result = normalize_path(path, base); + + //* Then + let normalized = result.expect("normalization should succeed"); + assert_eq!(normalized, "manifest.json"); + } + + #[test] + fn normalize_path_with_path_outside_base_fails() { + //* Given + let base = Path::new("/project"); + let path = Path::new("/other/file.sql"); + + //* When + let result = normalize_path(path, base); + + //* Then + assert!(result.is_err(), "should fail for path outside base"); + let err = result.expect_err("should return error"); + assert!( + matches!(err, PathError::NotWithinBase { .. }), + "should be NotWithinBase error" + ); + } + + #[test] + fn normalize_path_with_path_equal_to_base_fails() { + //* Given + let base = Path::new("/project"); + let path = Path::new("/project"); + + //* When + let result = normalize_path(path, base); + + //* Then + assert!(result.is_err(), "should fail when path equals base"); + let err = result.expect_err("should return error"); + assert!( + matches!(err, PathError::NotWithinBase { .. }), + "should be NotWithinBase error" + ); + } + + #[test] + fn normalize_path_rejects_dotdot_components() { + //* Given + let base = Path::new("/project"); + // This path technically resolves within base, but contains .. + let path = Path::new("/project/tables/../tables/users.sql"); + + //* When + let result = normalize_path(path, base); + + //* Then + // The path contains .. so it should be rejected + assert!(result.is_err(), "should reject paths with .."); + } + + #[test] + fn normalize_path_result_does_not_start_with_dot_slash() { + //* Given + let base = Path::new("/project"); + let path = Path::new("/project/file.sql"); + + //* When + let result = normalize_path(path, base); + + //* Then + let normalized = result.expect("normalization should succeed"); + assert!( + !normalized.starts_with("./"), + "result should not start with ./" + ); + assert!( + !normalized.starts_with('.'), + "result should not start with ." + ); + } + + #[test] + fn normalize_path_result_is_not_absolute() { + //* Given + let base = Path::new("/project"); + let path = Path::new("/project/tables/users.sql"); + + //* When + let result = normalize_path(path, base); + + //* Then + let normalized = result.expect("normalization should succeed"); + assert!( + !normalized.starts_with('/'), + "result should not be absolute" + ); + } + + // ============================================================ + // FileRef tests + // ============================================================ + + #[test] + fn file_ref_new_creates_instance() { + //* Given + let path = "tables/users.sql".to_string(); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("should parse hash"); + + //* When + let file_ref = FileRef::new(path.clone(), hash.clone()); + + //* Then + assert_eq!(file_ref.path, path); + assert_eq!(file_ref.hash, hash); + } + + #[test] + fn file_ref_from_file_creates_instance() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let tables_dir = dir.path().join("tables"); + std::fs::create_dir(&tables_dir).expect("should create tables dir"); + let file_path = tables_dir.join("users.sql"); + std::fs::write(&file_path, b"SELECT * FROM users").expect("should write file"); + + //* When + let result = FileRef::from_file(&file_path, dir.path()); + + //* Then + let file_ref = result.expect("should create file ref"); + assert_eq!(file_ref.path, "tables/users.sql"); + // Verify hash is computed correctly + let expected_hash = hash_file(&file_path).expect("should hash file"); + assert_eq!(file_ref.hash, expected_hash); + } + + #[test] + fn file_ref_from_file_with_nonexistent_file_returns_error() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let file_path = dir.path().join("nonexistent.sql"); + + //* When + let result = FileRef::from_file(&file_path, dir.path()); + + //* Then + assert!(result.is_err(), "should fail for nonexistent file"); + let err = result.expect_err("should return error"); + assert!( + matches!(err, FileRefError::HashFile { .. }), + "should be HashFile error" + ); + } + + #[test] + fn file_ref_serializes_to_json() { + //* Given + let file_ref = FileRef { + path: "tables/users.sql".to_string(), + hash: "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("should parse hash"), + }; + + //* When + let json = serde_json::to_string(&file_ref); + + //* Then + let json_str = json.expect("serialization should succeed"); + assert!(json_str.contains("tables/users.sql")); + assert!( + json_str.contains("b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9") + ); + } + + #[test] + fn file_ref_deserializes_from_json() { + //* Given + let json = r#"{"path":"tables/users.sql","hash":"b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"}"#; + + //* When + let result: Result = serde_json::from_str(json); + + //* Then + let file_ref = result.expect("deserialization should succeed"); + assert_eq!(file_ref.path, "tables/users.sql"); + assert_eq!( + file_ref.hash.as_str(), + "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + ); + } + + #[test] + fn file_ref_roundtrip_serialization() { + //* Given + let original = FileRef { + path: "functions/decode.js".to_string(), + hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .parse() + .expect("should parse hash"), + }; + + //* When + let json = serde_json::to_string(&original).expect("serialization should succeed"); + let deserialized: FileRef = + serde_json::from_str(&json).expect("deserialization should succeed"); + + //* Then + assert_eq!(deserialized, original); + } +} diff --git a/crates/core/dataset-authoring/src/jinja.rs b/crates/core/dataset-authoring/src/jinja.rs new file mode 100644 index 000000000..3d7e7e545 --- /dev/null +++ b/crates/core/dataset-authoring/src/jinja.rs @@ -0,0 +1,680 @@ +//! Jinja2 SQL templating. +//! +//! Provides sandboxed Jinja rendering with `ref`, `source`, `var`, `env_var`, +//! and `this` helpers for deterministic SQL template expansion. +//! +//! # Determinism +//! +//! The Jinja environment is strictly sandboxed: +//! - No `include` or `import` statements +//! - No filesystem or network access +//! - Variable resolution is fully deterministic given the same context +//! +//! # Functions +//! +//! - `ref(alias, table)` / `source(alias, table)` - Returns `".
"` for referencing +//! dependency tables. Validates that the alias exists in the declared dependencies. +//! - `var(name, default=None)` - Returns variable value. Precedence: CLI `--var` > amp.yaml `vars` > default. +//! Errors if missing without default. +//! - `env_var(name, default=None)` - Returns environment variable value. Precedence: env > default. +//! Errors if missing without default. +//! - `this` - Global variable containing the current table name. +//! +//! # Example +//! +//! ```ignore +//! use dataset_authoring::jinja::{JinjaContext, render_sql}; +//! +//! let ctx = JinjaContext::builder() +//! .with_cli_var("network", "mainnet") +//! .with_yaml_var("api_key", "default_key") +//! .with_dep_alias("eth") +//! .build(); +//! +//! let sql = r#" +//! SELECT * FROM {{ ref("eth", "blocks") }} +//! WHERE network = '{{ var("network") }}' +//! "#; +//! +//! let rendered = render_sql(sql, "my_table", &ctx)?; +//! // SELECT * FROM eth.blocks WHERE network = 'mainnet' +//! ``` + +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; + +use datasets_derived::deps::DepAlias; +use minijinja::{Environment, Error as JinjaError, ErrorKind, State}; + +/// Context for Jinja template rendering. +/// +/// Contains all variables and dependency information needed to render SQL templates. +/// Construct using [`JinjaContextBuilder`]. +#[derive(Debug, Clone)] +pub struct JinjaContext { + /// Variables passed via CLI `--var` arguments (highest priority). + cli_vars: BTreeMap, + /// Variables defined in amp.yaml `vars` section. + yaml_vars: BTreeMap, + /// Set of valid dependency aliases from amp.yaml `dependencies`. + dep_aliases: BTreeSet, +} + +impl JinjaContext { + /// Create a new builder for constructing a `JinjaContext`. + pub fn builder() -> JinjaContextBuilder { + JinjaContextBuilder::default() + } + + /// Get a variable value by precedence: CLI > YAML > default. + /// + /// Returns `None` if the variable is not found in either CLI or YAML vars. + fn get_var(&self, name: &str) -> Option<&str> { + self.cli_vars + .get(name) + .or_else(|| self.yaml_vars.get(name)) + .map(String::as_str) + } + + /// Check if a dependency alias is valid. + fn has_dep_alias(&self, alias: &str) -> bool { + // Try to parse as DepAlias and check if it exists + alias + .parse::() + .is_ok_and(|a| self.dep_aliases.contains(&a)) + } +} + +/// Builder for [`JinjaContext`]. +#[derive(Debug, Default)] +pub struct JinjaContextBuilder { + cli_vars: BTreeMap, + yaml_vars: BTreeMap, + dep_aliases: BTreeSet, +} + +impl JinjaContextBuilder { + /// Add a CLI variable (highest priority). + pub fn with_cli_var(mut self, name: impl Into, value: impl Into) -> Self { + self.cli_vars.insert(name.into(), value.into()); + self + } + + /// Add multiple CLI variables. + pub fn with_cli_vars(mut self, vars: impl IntoIterator) -> Self { + self.cli_vars.extend(vars); + self + } + + /// Add a YAML variable (from amp.yaml `vars` section). + pub fn with_yaml_var(mut self, name: impl Into, value: impl Into) -> Self { + self.yaml_vars.insert(name.into(), value.into()); + self + } + + /// Add multiple YAML variables. + pub fn with_yaml_vars(mut self, vars: impl IntoIterator) -> Self { + self.yaml_vars.extend(vars); + self + } + + /// Add a valid dependency alias. + pub fn with_dep_alias(mut self, alias: DepAlias) -> Self { + self.dep_aliases.insert(alias); + self + } + + /// Add multiple dependency aliases. + pub fn with_dep_aliases(mut self, aliases: impl IntoIterator) -> Self { + self.dep_aliases.extend(aliases); + self + } + + /// Build the context. + pub fn build(self) -> JinjaContext { + JinjaContext { + cli_vars: self.cli_vars, + yaml_vars: self.yaml_vars, + dep_aliases: self.dep_aliases, + } + } +} + +/// Errors that occur during Jinja template rendering. +#[derive(Debug, thiserror::Error)] +pub enum RenderError { + /// Template rendering failed. + #[error("failed to render template")] + Render { + /// The underlying minijinja error. + #[source] + source: JinjaError, + }, + + /// Template compilation failed. + #[error("invalid template syntax")] + Template { + /// The underlying minijinja error. + #[source] + source: JinjaError, + }, +} + +/// Render a SQL template with the given context and table name. +/// +/// The template has access to: +/// - `ref(alias, table)` - Reference a dependency table +/// - `source(alias, table)` - Alias for `ref` +/// - `var(name, default=None)` - Get a variable value +/// - `env_var(name, default=None)` - Get an environment variable +/// - `this` - The current table name +/// +/// # Errors +/// +/// Returns [`RenderError::Template`] if the template syntax is invalid. +/// Returns [`RenderError::Render`] if rendering fails (e.g., missing required variable). +pub fn render_sql( + template: &str, + table_name: &str, + context: &JinjaContext, +) -> Result { + let env = create_environment(context); + + // Add the template + let compiled = env + .template_from_str(template) + .map_err(|source| RenderError::Template { source })?; + + // Create render context with `this` set to current table name + let ctx = minijinja::context! { + this => table_name, + }; + + compiled + .render(ctx) + .map_err(|source| RenderError::Render { source }) +} + +/// Create a sandboxed minijinja environment with custom functions. +/// +/// The environment has auto-escape disabled (SQL doesn't need HTML escaping) +/// and only allows our deterministic helper functions. +fn create_environment(context: &JinjaContext) -> Environment<'static> { + let mut env = Environment::new(); + + // Disable auto-escaping for SQL templates + env.set_auto_escape_callback(|_| minijinja::AutoEscape::None); + + // Register the `ref` function + let ctx = Arc::new(context.clone()); + let ref_ctx = ctx.clone(); + env.add_function("ref", move |alias: &str, table: &str| -> Result { + if !ref_ctx.has_dep_alias(alias) { + return Err(JinjaError::new( + ErrorKind::InvalidOperation, + format!("unknown dependency alias '{alias}'; must be declared in amp.yaml dependencies"), + )); + } + Ok(format!("{alias}.{table}")) + }); + + // Register `source` as an alias for `ref` + let source_ctx = ctx.clone(); + env.add_function( + "source", + move |alias: &str, table: &str| -> Result { + if !source_ctx.has_dep_alias(alias) { + return Err(JinjaError::new( + ErrorKind::InvalidOperation, + format!( + "unknown dependency alias '{alias}'; must be declared in amp.yaml dependencies" + ), + )); + } + Ok(format!("{alias}.{table}")) + }, + ); + + // Register the `var` function + let var_ctx = ctx.clone(); + env.add_function( + "var", + move |_state: &State, name: &str, default: Option<&str>| -> Result { + match var_ctx.get_var(name) { + Some(value) => Ok(value.to_string()), + None => match default { + Some(d) => Ok(d.to_string()), + None => Err(JinjaError::new( + ErrorKind::UndefinedError, + format!("variable '{name}' is not defined and no default was provided"), + )), + }, + } + }, + ); + + // Register the `env_var` function + env.add_function( + "env_var", + |_state: &State, name: &str, default: Option<&str>| -> Result { + match std::env::var(name) { + Ok(value) => Ok(value), + Err(_) => match default { + Some(d) => Ok(d.to_string()), + None => Err(JinjaError::new( + ErrorKind::UndefinedError, + format!( + "environment variable '{name}' is not set and no default was provided" + ), + )), + }, + } + }, + ); + + env +} + +#[cfg(test)] +mod tests { + use super::*; + + mod ref_function { + use super::*; + + #[test] + fn ref_with_valid_alias_returns_qualified_name() { + //* Given + let ctx = JinjaContext::builder() + .with_dep_alias("eth".parse().expect("valid alias")) + .build(); + let template = r#"{{ ref("eth", "blocks") }}"#; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!(rendered, "eth.blocks"); + } + + #[test] + fn ref_with_invalid_alias_fails() { + //* Given + let ctx = JinjaContext::builder().build(); + let template = r#"{{ ref("unknown", "blocks") }}"#; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let err = result.expect_err("should fail with unknown alias"); + assert!(matches!(err, RenderError::Render { .. })); + let message = err.to_string(); + assert!( + message.contains("render"), + "error should be about rendering: {message}" + ); + } + + #[test] + fn source_is_alias_for_ref() { + //* Given + let ctx = JinjaContext::builder() + .with_dep_alias("eth".parse().expect("valid alias")) + .build(); + let template = r#"{{ source("eth", "transactions") }}"#; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!(rendered, "eth.transactions"); + } + + #[test] + fn source_with_invalid_alias_fails() { + //* Given + let ctx = JinjaContext::builder().build(); + let template = r#"{{ source("missing", "table") }}"#; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let err = result.expect_err("should fail with unknown alias"); + assert!(matches!(err, RenderError::Render { .. })); + } + } + + mod var_function { + use super::*; + + #[test] + fn var_with_cli_value_returns_cli_value() { + //* Given + let ctx = JinjaContext::builder() + .with_cli_var("network", "mainnet") + .with_yaml_var("network", "testnet") + .build(); + let template = r#"{{ var("network") }}"#; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!(rendered, "mainnet", "CLI should take precedence over YAML"); + } + + #[test] + fn var_with_yaml_value_returns_yaml_value() { + //* Given + let ctx = JinjaContext::builder() + .with_yaml_var("api_key", "default_key") + .build(); + let template = r#"{{ var("api_key") }}"#; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!(rendered, "default_key"); + } + + #[test] + fn var_with_default_returns_default_when_missing() { + //* Given + let ctx = JinjaContext::builder().build(); + let template = r#"{{ var("missing", "fallback") }}"#; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!(rendered, "fallback"); + } + + #[test] + fn var_without_default_fails_when_missing() { + //* Given + let ctx = JinjaContext::builder().build(); + let template = r#"{{ var("missing") }}"#; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let err = result.expect_err("should fail with missing var"); + assert!(matches!(err, RenderError::Render { .. })); + } + + #[test] + fn var_precedence_cli_over_yaml_over_default() { + //* Given - test that CLI wins over YAML + let ctx = JinjaContext::builder() + .with_cli_var("priority_test", "cli_wins") + .with_yaml_var("priority_test", "yaml_value") + .build(); + let template = r#"{{ var("priority_test", "default_value") }}"#; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!(rendered, "cli_wins"); + } + + #[test] + fn var_precedence_yaml_over_default() { + //* Given - YAML present, no CLI + let ctx = JinjaContext::builder() + .with_yaml_var("yaml_only", "yaml_value") + .build(); + let template = r#"{{ var("yaml_only", "default_value") }}"#; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!(rendered, "yaml_value"); + } + } + + mod env_var_function { + use super::*; + + #[test] + fn env_var_with_set_value_returns_value() { + //* Given + // Use a var that should always be set + let template = r#"{{ env_var("PATH", "not_set") }}"#; + let ctx = JinjaContext::builder().build(); + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_ne!(rendered, "not_set", "PATH should be set in environment"); + } + + #[test] + fn env_var_with_default_returns_default_when_unset() { + //* Given + let template = + r#"{{ env_var("AMP_NONEXISTENT_VAR_FOR_TESTING_12345", "fallback_default") }}"#; + let ctx = JinjaContext::builder().build(); + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!(rendered, "fallback_default"); + } + + #[test] + fn env_var_without_default_fails_when_unset() { + //* Given + let template = r#"{{ env_var("AMP_NONEXISTENT_VAR_FOR_TESTING_12345") }}"#; + let ctx = JinjaContext::builder().build(); + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let err = result.expect_err("should fail with missing env var"); + assert!(matches!(err, RenderError::Render { .. })); + } + } + + mod this_variable { + use super::*; + + #[test] + fn this_returns_current_table_name() { + //* Given + let ctx = JinjaContext::builder().build(); + let template = r#"CREATE TABLE {{ this }} AS SELECT 1"#; + + //* When + let result = render_sql(template, "my_table", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!(rendered, "CREATE TABLE my_table AS SELECT 1"); + } + + #[test] + fn this_works_in_complex_template() { + //* Given + let ctx = JinjaContext::builder() + .with_dep_alias("eth".parse().expect("valid alias")) + .build(); + let template = r#"CREATE TABLE {{ this }} AS SELECT * FROM {{ ref("eth", "blocks") }} WHERE table_name = '{{ this }}'"#; + + //* When + let result = render_sql(template, "block_stats", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!( + rendered, + "CREATE TABLE block_stats AS SELECT * FROM eth.blocks WHERE table_name = 'block_stats'" + ); + } + } + + mod template_syntax { + use super::*; + + #[test] + fn invalid_template_syntax_fails() { + //* Given + let ctx = JinjaContext::builder().build(); + let template = r#"{{ unclosed"#; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let err = result.expect_err("should fail with invalid syntax"); + assert!(matches!(err, RenderError::Template { .. })); + } + + #[test] + fn plain_sql_without_jinja_passes_through() { + //* Given + let ctx = JinjaContext::builder().build(); + let template = "SELECT * FROM some_table WHERE id = 1"; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!(rendered, template); + } + + #[test] + fn multiline_template_renders_correctly() { + //* Given + let ctx = JinjaContext::builder() + .with_dep_alias("eth".parse().expect("valid alias")) + .with_cli_var("min_value", "1000") + .build(); + let template = r#"SELECT + block_number, + tx_hash +FROM {{ ref("eth", "transactions") }} +WHERE value > {{ var("min_value") }}"#; + + //* When + let result = render_sql(template, "test_table", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + let expected = r#"SELECT + block_number, + tx_hash +FROM eth.transactions +WHERE value > 1000"#; + assert_eq!(rendered, expected); + } + } + + mod context_builder { + use super::*; + + #[test] + fn builder_accepts_multiple_deps() { + //* Given + let aliases = ["eth", "polygon", "base"] + .iter() + .map(|s| s.parse::().expect("valid alias")); + + //* When + let ctx = JinjaContext::builder().with_dep_aliases(aliases).build(); + + //* Then + assert!(ctx.has_dep_alias("eth")); + assert!(ctx.has_dep_alias("polygon")); + assert!(ctx.has_dep_alias("base")); + assert!(!ctx.has_dep_alias("unknown")); + } + + #[test] + fn builder_accepts_multiple_cli_vars() { + //* Given + let vars = vec![ + ("var1".to_string(), "value1".to_string()), + ("var2".to_string(), "value2".to_string()), + ]; + + //* When + let ctx = JinjaContext::builder().with_cli_vars(vars).build(); + let template = r#"{{ var("var1") }}-{{ var("var2") }}"#; + let result = render_sql(template, "test", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!(rendered, "value1-value2"); + } + + #[test] + fn builder_accepts_multiple_yaml_vars() { + //* Given + let vars = vec![ + ("yaml1".to_string(), "yval1".to_string()), + ("yaml2".to_string(), "yval2".to_string()), + ]; + + //* When + let ctx = JinjaContext::builder().with_yaml_vars(vars).build(); + let template = r#"{{ var("yaml1") }}-{{ var("yaml2") }}"#; + let result = render_sql(template, "test", &ctx); + + //* Then + let rendered = result.expect("should render successfully"); + assert_eq!(rendered, "yval1-yval2"); + } + } + + mod determinism { + use super::*; + + #[test] + fn rendering_is_deterministic() { + //* Given + let ctx = JinjaContext::builder() + .with_dep_alias("eth".parse().expect("valid alias")) + .with_cli_var("network", "mainnet") + .with_yaml_var("default_limit", "100") + .build(); + let template = r#"SELECT * FROM {{ ref("eth", "blocks") }} WHERE network = '{{ var("network") }}' LIMIT {{ var("default_limit") }}"#; + + //* When - render multiple times + let results: Vec<_> = (0..5) + .map(|_| render_sql(template, "test_table", &ctx)) + .collect(); + + //* Then + let first = results[0] + .as_ref() + .expect("should render successfully") + .clone(); + for result in &results[1..] { + let rendered = result.as_ref().expect("should render successfully"); + assert_eq!(&first, rendered, "all renders should be identical"); + } + } + } +} diff --git a/crates/core/dataset-authoring/src/lib.rs b/crates/core/dataset-authoring/src/lib.rs new file mode 100644 index 000000000..46f489175 --- /dev/null +++ b/crates/core/dataset-authoring/src/lib.rs @@ -0,0 +1,49 @@ +//! Dataset authoring support for Amp. +//! +//! This crate provides the authoring workflow for derived datasets, including: +//! +//! - **Configuration**: `amp.yaml` parsing and validation ([`config`]) +//! - **Table discovery**: dbt-style table file discovery ([`discovery`]) +//! - **Templating**: Jinja2-compatible SQL templating ([`jinja`]) +//! - **SQL validation**: SELECT statement validation ([`query`]) +//! - **Schema inference**: Arrow schema inference via DataFusion planning ([`schema`]) +//! - **Schema files (IPC)**: Arrow schema IPC serialization ([`arrow_ipc`]) +//! - **Schema files (JSON)**: Arrow schema JSON serialization ([`arrow_json`]) +//! - **File utilities**: File hashing and path normalization ([`files`]) +//! - **Manifest generation**: Canonical JSON manifest with content hashing ([`manifest`]) +//! - **Packaging**: Deterministic archive creation for deployment ([`package`]) +//! - **Caching**: Global cache for resolved dependencies ([`cache`]) +//! - **Lockfile**: Dependency lockfile for reproducible builds ([`lockfile`]) +//! - **Legacy bridge**: Conversion to legacy inline manifest format ([`bridge`]) +//! - **Legacy adapter**: Conversion from legacy inline manifest to package format ([`adapter`]) +//! +//! ## Authoring Workflow +//! +//! 1. Parse `amp.yaml` configuration +//! 2. Discover tables from `tables/` directory +//! 3. Resolve dependencies from registry +//! 4. Render Jinja SQL templates +//! 5. Validate SELECT statements +//! 6. Infer schemas via DataFusion +//! 7. Write schema files to `tables/
.ipc` +//! 8. Generate canonical manifest with file hashes +//! 9. Package for deployment + +pub mod adapter; +pub mod arrow_ipc; +pub mod arrow_json; +pub mod bridge; +pub mod cache; +pub mod canonical; +pub mod config; +pub mod dependency_manifest; +pub mod discovery; +pub mod files; +pub mod jinja; +pub mod lockfile; +pub mod manifest; +pub mod package; +pub mod query; +pub mod resolver; +pub mod schema; +pub mod validation; diff --git a/crates/core/dataset-authoring/src/lockfile.rs b/crates/core/dataset-authoring/src/lockfile.rs new file mode 100644 index 000000000..8732e37b5 --- /dev/null +++ b/crates/core/dataset-authoring/src/lockfile.rs @@ -0,0 +1,1201 @@ +//! Dependency lockfile support. +//! +//! Defines the `amp.lock` format for reproducible builds. The lockfile captures +//! the complete resolved dependency graph at a point in time, enabling: +//! +//! - **Reproducible builds**: Same lockfile = same dependency versions +//! - **Offline resolution**: Lockfile contains all information needed for builds +//! - **Change detection**: Diff lockfiles to see dependency changes +//! +//! # Lockfile Format +//! +//! The lockfile is a YAML file with the following structure: +//! +//! ```yaml +//! version: 1 +//! root: +//! namespace: my_namespace +//! name: my_dataset +//! version: 1.0.0 +//! dependencies: +//! eth: b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 +//! tokens: a1b2c3d4e5f6... +//! nodes: +//! b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9: +//! namespace: graph +//! name: eth_mainnet +//! deps: +//! - cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +//! ``` +//! +//! # Usage Modes +//! +//! - **Normal**: Resolve dependencies, update lockfile if changed +//! - **`--locked`**: Fail if lockfile is missing or out of date + +use std::{ + collections::{BTreeMap, BTreeSet}, + io, + path::{Path, PathBuf}, +}; + +use datasets_common::{hash::Hash, name::Name, namespace::Namespace, version::Version}; +use datasets_derived::deps::{DepAlias, DepReference, HashOrVersion}; + +use crate::resolver::DependencyGraph; + +/// Current lockfile format version. +pub const LOCKFILE_VERSION: u32 = 1; + +/// Default lockfile filename. +pub const LOCKFILE_NAME: &str = "amp.lock"; + +/// Root package information in the lockfile. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct RootInfo { + /// Package namespace. + pub namespace: Namespace, + /// Package name. + pub name: Name, + /// Package version. + pub version: Version, +} + +/// Information about a dependency node in the lockfile. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct NodeInfo { + /// Dependency namespace. + pub namespace: String, + /// Dependency name. + pub name: String, + /// Hashes of this node's direct dependencies. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub deps: Vec, +} + +/// The amp.lock lockfile for dependency resolution. +/// +/// The lockfile captures the complete resolved dependency graph at a point +/// in time, enabling reproducible builds. It contains: +/// +/// - Format version for compatibility checking +/// - Root package information +/// - Direct dependency aliases mapped to their resolved hashes +/// - All transitive dependency nodes keyed by hash +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Lockfile { + /// Lockfile format version. Always 1 for this implementation. + pub version: u32, + /// Information about the root package being built. + pub root: RootInfo, + /// Direct dependencies mapped from alias to resolved hash. + pub dependencies: BTreeMap, + /// All transitive dependency nodes keyed by content hash. + pub nodes: BTreeMap, +} + +impl Lockfile { + /// Creates a new lockfile from root info and a resolved dependency graph. + /// + /// This extracts the necessary information from the dependency graph to + /// create a lockfile that can reproduce the same resolution. + pub fn from_graph(root: RootInfo, graph: &DependencyGraph) -> Self { + let mut nodes = BTreeMap::new(); + + for (hash, node) in &graph.nodes { + // Extract namespace and name from the resolved dependency + // These values were captured during dependency resolution from the DepReference. + let namespace = node.resolved.namespace.to_string(); + let name = node.resolved.name.to_string(); + + nodes.insert( + hash.clone(), + NodeInfo { + namespace, + name, + deps: node.deps.clone(), + }, + ); + } + + Self { + version: LOCKFILE_VERSION, + root, + dependencies: graph.direct.clone(), + nodes, + } + } + + /// Reads a lockfile from the given path. + /// + /// # Errors + /// + /// Returns an error if the file cannot be read or parsed. + pub fn read(path: &Path) -> Result { + let contents = + std::fs::read_to_string(path).map_err(|err| LockfileError::ReadFile { source: err })?; + + Self::from_yaml(&contents) + } + + /// Parses a lockfile from a YAML string. + /// + /// # Errors + /// + /// Returns an error if the YAML is invalid or doesn't match the expected schema. + pub fn from_yaml(yaml: &str) -> Result { + let lockfile: Lockfile = + serde_yaml::from_str(yaml).map_err(|err| LockfileError::Parse { source: err })?; + + // Validate version + if lockfile.version != LOCKFILE_VERSION { + return Err(LockfileError::UnsupportedVersion { + found: lockfile.version, + expected: LOCKFILE_VERSION, + }); + } + + Ok(lockfile) + } + + /// Serializes the lockfile to a YAML string. + /// + /// The output is deterministic: same lockfile contents = same YAML bytes. + /// This is achieved through BTreeMap's sorted iteration. + /// + /// # Errors + /// + /// Returns an error if serialization fails. + pub fn to_yaml(&self) -> Result { + serde_yaml::to_string(self).map_err(|err| LockfileError::Serialize { source: err }) + } + + /// Writes the lockfile to the given path. + /// + /// # Errors + /// + /// Returns an error if the file cannot be written. + pub fn write(&self, path: &Path) -> Result<(), LockfileError> { + let yaml = self.to_yaml()?; + std::fs::write(path, yaml).map_err(|err| LockfileError::WriteFile { source: err })?; + Ok(()) + } + + /// Checks if this lockfile's root metadata matches the expected root info. + /// + /// # Errors + /// + /// Returns [`LockfileError::RootMismatch`] if the root metadata does not match. + pub fn verify_root(&self, root: &RootInfo) -> Result<(), LockfileError> { + if self.root != *root { + return Err(LockfileError::RootMismatch { + lockfile: Box::new(self.root.clone()), + expected: Box::new(root.clone()), + }); + } + + Ok(()) + } + + /// Checks if this lockfile is compatible with a resolved dependency graph. + /// + /// Returns `Ok(())` if the lockfile matches the graph, or an error describing + /// the mismatch. + /// + /// This is used for `--locked` mode to verify the lockfile is up to date. + pub fn verify(&self, graph: &DependencyGraph) -> Result<(), LockfileError> { + self.verify_with_root(None, graph) + } + + /// Checks if this lockfile is compatible with a resolved dependency graph and root info. + /// + /// Returns `Ok(())` if the lockfile matches the graph and root info, or an error describing + /// the mismatch. + /// + /// This is used for `--locked` mode to verify the lockfile is up to date and matches + /// the current package metadata. + pub fn verify_with_root( + &self, + root: Option<&RootInfo>, + graph: &DependencyGraph, + ) -> Result<(), LockfileError> { + // Check root metadata matches if provided + if let Some(root) = root { + self.verify_root(root)?; + } + + // Check direct dependencies match + if self.dependencies != graph.direct { + return Err(LockfileError::DirectDepsMismatch { + lockfile: self.dependencies.keys().cloned().collect(), + resolved: graph.direct.keys().cloned().collect(), + }); + } + + // Check all nodes are present + for (hash, node) in &graph.nodes { + if !self.nodes.contains_key(hash) { + return Err(LockfileError::MissingNode { hash: hash.clone() }); + } + + // Check deps match + let locked_node = &self.nodes[hash]; + if locked_node.deps != node.deps { + return Err(LockfileError::NodeDepsMismatch { hash: hash.clone() }); + } + } + + // Check no extra nodes in lockfile + for hash in self.nodes.keys() { + if !graph.nodes.contains_key(hash) { + return Err(LockfileError::ExtraNode { hash: hash.clone() }); + } + } + + Ok(()) + } + + /// Checks if this lockfile is compatible with the direct dependencies + /// declared in `amp.yaml`. + /// + /// This validates: + /// - Direct dependency aliases match between config and lockfile. + /// - Hash references in config match the lockfile hash. + /// - Namespace/name in lockfile matches the config reference FQN. + /// - Lockfile node dependencies reference existing nodes. + /// + /// # Errors + /// + /// Returns [`LockfileError`] variants describing any mismatch. + pub fn verify_dependencies( + &self, + dependencies: &BTreeMap, + ) -> Result<(), LockfileError> { + let lockfile_aliases: BTreeSet = self.dependencies.keys().cloned().collect(); + let config_aliases: BTreeSet = dependencies.keys().cloned().collect(); + + if lockfile_aliases != config_aliases { + return Err(LockfileError::DirectDepsMismatch { + lockfile: lockfile_aliases.into_iter().collect(), + resolved: config_aliases.into_iter().collect(), + }); + } + + for (alias, reference) in dependencies { + let locked_hash = + self.dependencies + .get(alias) + .ok_or_else(|| LockfileError::DirectDepsMismatch { + lockfile: self.dependencies.keys().cloned().collect(), + resolved: dependencies.keys().cloned().collect(), + })?; + + if let HashOrVersion::Hash(expected_hash) = reference.revision() + && expected_hash != locked_hash + { + return Err(LockfileError::DirectDepHashMismatch { + alias: alias.clone(), + lockfile: locked_hash.clone(), + config: expected_hash.clone(), + }); + } + + let Some(node) = self.nodes.get(locked_hash) else { + return Err(LockfileError::MissingNode { + hash: locked_hash.clone(), + }); + }; + + let expected_namespace = reference.fqn().namespace().as_str(); + let expected_name = reference.fqn().name().as_str(); + + if node.namespace != expected_namespace || node.name != expected_name { + return Err(LockfileError::DirectDepFqnMismatch { + alias: alias.clone(), + lockfile_namespace: node.namespace.clone(), + lockfile_name: node.name.clone(), + expected_namespace: expected_namespace.to_string(), + expected_name: expected_name.to_string(), + }); + } + } + + for node in self.nodes.values() { + for dep in &node.deps { + if !self.nodes.contains_key(dep) { + return Err(LockfileError::MissingNode { hash: dep.clone() }); + } + } + } + + let mut reachable: BTreeSet = BTreeSet::new(); + let mut pending: Vec = self.dependencies.values().cloned().collect(); + while let Some(hash) = pending.pop() { + if !reachable.insert(hash.clone()) { + continue; + } + + let node = self + .nodes + .get(&hash) + .ok_or_else(|| LockfileError::MissingNode { hash: hash.clone() })?; + for dep in &node.deps { + pending.push(dep.clone()); + } + } + + for hash in self.nodes.keys() { + if !reachable.contains(hash) { + return Err(LockfileError::ExtraNode { hash: hash.clone() }); + } + } + + Ok(()) + } + + /// Returns the path to the lockfile in the given directory. + pub fn path_in(dir: &Path) -> PathBuf { + dir.join(LOCKFILE_NAME) + } +} + +/// Errors that occur during lockfile operations. +#[derive(Debug, thiserror::Error)] +pub enum LockfileError { + /// Failed to read lockfile from disk. + /// + /// This occurs when the lockfile cannot be read, typically due to + /// missing file, permission issues, or I/O errors. + #[error("failed to read lockfile")] + ReadFile { + #[source] + source: io::Error, + }, + + /// Failed to parse lockfile YAML. + /// + /// This occurs when the lockfile exists but contains invalid YAML + /// or doesn't match the expected schema. + #[error("failed to parse lockfile")] + Parse { + #[source] + source: serde_yaml::Error, + }, + + /// Unsupported lockfile version. + /// + /// This occurs when the lockfile was created with a newer format + /// that this version doesn't support. + #[error("unsupported lockfile version: found {found}, expected {expected}")] + UnsupportedVersion { + /// The version found in the lockfile. + found: u32, + /// The version expected by this implementation. + expected: u32, + }, + + /// Failed to serialize lockfile to YAML. + /// + /// This is an internal error that should rarely occur. + #[error("failed to serialize lockfile")] + Serialize { + #[source] + source: serde_yaml::Error, + }, + + /// Failed to write lockfile to disk. + /// + /// This occurs when the lockfile cannot be written, typically due to + /// permission issues or disk space constraints. + #[error("failed to write lockfile")] + WriteFile { + #[source] + source: io::Error, + }, + + /// Lockfile is missing (required for --locked mode). + /// + /// This occurs in --locked mode when no lockfile exists. + #[error("lockfile not found (required for --locked mode)")] + NotFound, + + /// Root package metadata mismatch. + /// + /// This occurs in --locked mode when the lockfile's root metadata + /// (namespace, name, version) doesn't match the current amp.yaml. + #[error( + "root mismatch: lockfile has {}/{} v{}, expected {}/{} v{}", + lockfile.namespace, lockfile.name, lockfile.version, + expected.namespace, expected.name, expected.version + )] + RootMismatch { + /// Root info in the lockfile. + lockfile: Box, + /// Expected root info from amp.yaml. + expected: Box, + }, + + /// Direct dependencies in lockfile don't match resolved dependencies. + /// + /// This occurs in --locked mode when the lockfile's direct dependencies + /// don't match what was resolved from amp.yaml. + #[error("direct dependencies mismatch: lockfile has {lockfile:?}, resolved has {resolved:?}")] + DirectDepsMismatch { + /// Aliases present in the lockfile. + lockfile: Vec, + /// Aliases present in the resolved graph. + resolved: Vec, + }, + + /// Direct dependency hash mismatch. + /// + /// This occurs in --locked mode when amp.yaml specifies a hash reference + /// that differs from the hash recorded in amp.lock. + #[error( + "direct dependency '{alias}' hash mismatch: lockfile has {lockfile}, config has {config}" + )] + DirectDepHashMismatch { + /// The dependency alias with mismatched hashes. + alias: DepAlias, + /// The hash recorded in the lockfile. + lockfile: Hash, + /// The hash declared in amp.yaml. + config: Hash, + }, + + /// Direct dependency namespace/name mismatch. + /// + /// This occurs in --locked mode when amp.yaml references a different + /// dataset than the lockfile records for the alias. + #[error( + "direct dependency '{alias}' points to {lockfile_namespace}/{lockfile_name} in lockfile, expected {expected_namespace}/{expected_name}" + )] + DirectDepFqnMismatch { + /// The dependency alias with mismatched target. + alias: DepAlias, + /// Namespace recorded in amp.lock for the alias. + lockfile_namespace: String, + /// Name recorded in amp.lock for the alias. + lockfile_name: String, + /// Namespace declared in amp.yaml. + expected_namespace: String, + /// Name declared in amp.yaml. + expected_name: String, + }, + + /// A resolved dependency is missing from the lockfile. + /// + /// This occurs in --locked mode when a dependency was resolved that + /// doesn't exist in the lockfile. + #[error("dependency with hash '{hash}' not in lockfile")] + MissingNode { + /// The hash of the missing dependency. + hash: Hash, + }, + + /// A dependency's transitive deps don't match the lockfile. + /// + /// This occurs in --locked mode when a dependency's own dependencies + /// don't match what's recorded in the lockfile. + #[error("dependency '{hash}' has different deps than lockfile")] + NodeDepsMismatch { + /// The hash of the dependency with mismatched deps. + hash: Hash, + }, + + /// Lockfile contains a dependency not in the resolved graph. + /// + /// This occurs in --locked mode when the lockfile contains nodes + /// that are no longer part of the dependency graph. + #[error("lockfile contains extra dependency '{hash}' not in resolved graph")] + ExtraNode { + /// The hash of the extra dependency. + hash: Hash, + }, +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use datasets_common::dataset_kind_str::DatasetKindStr; + + use super::*; + use crate::{ + dependency_manifest::DependencyManifest, + resolver::{DependencyNode, ResolvedDependency}, + }; + + fn create_test_manifest() -> DependencyManifest { + DependencyManifest { + kind: DatasetKindStr::new("manifest".to_string()), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + } + } + + fn create_root_info() -> RootInfo { + RootInfo { + namespace: "test_namespace".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + version: "1.0.0".parse().expect("valid version"), + } + } + + /// Helper to create a test resolved dependency with default namespace/name. + fn create_test_resolved_dependency( + hash: Hash, + manifest: DependencyManifest, + ) -> ResolvedDependency { + ResolvedDependency { + hash, + manifest, + namespace: "test".parse().expect("valid namespace"), + name: "dataset".parse().expect("valid name"), + } + } + + #[test] + fn lockfile_version_is_one() { + assert_eq!(LOCKFILE_VERSION, 1); + } + + #[test] + fn lockfile_name_is_amp_lock() { + assert_eq!(LOCKFILE_NAME, "amp.lock"); + } + + #[test] + fn from_graph_creates_empty_lockfile_for_empty_graph() { + //* Given + let root = create_root_info(); + let graph = DependencyGraph::new(); + + //* When + let lockfile = Lockfile::from_graph(root.clone(), &graph); + + //* Then + assert_eq!(lockfile.version, LOCKFILE_VERSION); + assert_eq!(lockfile.root, root); + assert!(lockfile.dependencies.is_empty()); + assert!(lockfile.nodes.is_empty()); + } + + #[test] + fn from_graph_captures_direct_dependencies() { + //* Given + let root = create_root_info(); + let mut graph = DependencyGraph::new(); + + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let alias: DepAlias = "eth".parse().expect("valid alias"); + + graph.direct.insert(alias.clone(), hash.clone()); + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash.clone(), create_test_manifest()), + deps: vec![], + }, + ); + + //* When + let lockfile = Lockfile::from_graph(root, &graph); + + //* Then + assert_eq!(lockfile.dependencies.len(), 1); + assert_eq!(lockfile.dependencies.get(&alias), Some(&hash)); + assert_eq!(lockfile.nodes.len(), 1); + assert!(lockfile.nodes.contains_key(&hash)); + } + + #[test] + fn from_graph_captures_transitive_dependencies() { + //* Given + let root = create_root_info(); + let mut graph = DependencyGraph::new(); + + let hash_a: Hash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .parse() + .expect("valid hash"); + let hash_b: Hash = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + .parse() + .expect("valid hash"); + let alias: DepAlias = "dep_a".parse().expect("valid alias"); + + graph.direct.insert(alias.clone(), hash_a.clone()); + graph.nodes.insert( + hash_a.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash_a.clone(), create_test_manifest()), + deps: vec![hash_b.clone()], + }, + ); + graph.nodes.insert( + hash_b.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash_b.clone(), create_test_manifest()), + deps: vec![], + }, + ); + + //* When + let lockfile = Lockfile::from_graph(root, &graph); + + //* Then + assert_eq!(lockfile.nodes.len(), 2); + assert!(lockfile.nodes.contains_key(&hash_a)); + assert!(lockfile.nodes.contains_key(&hash_b)); + + let node_a = &lockfile.nodes[&hash_a]; + assert_eq!(node_a.deps, vec![hash_b.clone()]); + + let node_b = &lockfile.nodes[&hash_b]; + assert!(node_b.deps.is_empty()); + } + + #[test] + fn to_yaml_produces_valid_yaml() { + //* Given + let root = create_root_info(); + let graph = DependencyGraph::new(); + let lockfile = Lockfile::from_graph(root, &graph); + + //* When + let yaml = lockfile.to_yaml(); + + //* Then + assert!(yaml.is_ok(), "serialization should succeed"); + let yaml_str = yaml.expect("should have yaml"); + assert!(yaml_str.contains("version: 1")); + assert!(yaml_str.contains("namespace: test_namespace")); + assert!(yaml_str.contains("name: test_dataset")); + assert!(yaml_str.contains("version: 1.0.0")); + } + + #[test] + fn yaml_roundtrip_preserves_data() { + //* Given + let root = create_root_info(); + let mut graph = DependencyGraph::new(); + + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let alias: DepAlias = "eth".parse().expect("valid alias"); + + graph.direct.insert(alias, hash.clone()); + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash.clone(), create_test_manifest()), + deps: vec![], + }, + ); + + let original = Lockfile::from_graph(root, &graph); + + //* When + let yaml = original.to_yaml().expect("should serialize"); + let parsed = Lockfile::from_yaml(&yaml); + + //* Then + assert!(parsed.is_ok(), "parsing should succeed"); + let parsed = parsed.expect("should have lockfile"); + assert_eq!(parsed.version, original.version); + assert_eq!(parsed.root, original.root); + assert_eq!(parsed.dependencies, original.dependencies); + assert_eq!(parsed.nodes, original.nodes); + } + + #[test] + fn yaml_output_is_deterministic() { + //* Given + let root = create_root_info(); + let mut graph = DependencyGraph::new(); + + // Add multiple dependencies to test ordering + let hash_a: Hash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .parse() + .expect("valid hash"); + let hash_b: Hash = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + .parse() + .expect("valid hash"); + + let alias_a: DepAlias = "alpha".parse().expect("valid alias"); + let alias_b: DepAlias = "beta".parse().expect("valid alias"); + + graph.direct.insert(alias_a, hash_a.clone()); + graph.direct.insert(alias_b, hash_b.clone()); + graph.nodes.insert( + hash_a.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash_a.clone(), create_test_manifest()), + deps: vec![], + }, + ); + graph.nodes.insert( + hash_b.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash_b.clone(), create_test_manifest()), + deps: vec![], + }, + ); + + let lockfile = Lockfile::from_graph(root, &graph); + + //* When + let yaml1 = lockfile.to_yaml().expect("should serialize"); + let yaml2 = lockfile.to_yaml().expect("should serialize"); + + //* Then + assert_eq!(yaml1, yaml2, "YAML output should be deterministic"); + } + + #[test] + fn from_yaml_rejects_unsupported_version() { + //* Given + let yaml = r#" +version: 999 +root: + namespace: test_namespace + name: test_dataset + version: 1.0.0 +dependencies: {} +nodes: {} +"#; + + //* When + let result = Lockfile::from_yaml(yaml); + + //* Then + assert!(result.is_err(), "should reject unsupported version"); + let err = result.expect_err("should have error"); + assert!( + matches!(err, LockfileError::UnsupportedVersion { found: 999, .. }), + "should be UnsupportedVersion error, got: {:?}", + err + ); + } + + #[test] + fn from_yaml_rejects_invalid_yaml() { + //* Given + let yaml = "not: valid: yaml: [[["; + + //* When + let result = Lockfile::from_yaml(yaml); + + //* Then + assert!(result.is_err(), "should reject invalid YAML"); + let err = result.expect_err("should have error"); + assert!( + matches!(err, LockfileError::Parse { .. }), + "should be Parse error" + ); + } + + #[test] + fn verify_succeeds_for_matching_graph() { + //* Given + let root = create_root_info(); + let mut graph = DependencyGraph::new(); + + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let alias: DepAlias = "eth".parse().expect("valid alias"); + + graph.direct.insert(alias, hash.clone()); + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash.clone(), create_test_manifest()), + deps: vec![], + }, + ); + + let lockfile = Lockfile::from_graph(root, &graph); + + //* When + let result = lockfile.verify(&graph); + + //* Then + assert!( + result.is_ok(), + "verification should succeed for matching graph" + ); + } + + #[test] + fn verify_fails_for_missing_direct_dep() { + //* Given + let root = create_root_info(); + let graph1 = DependencyGraph::new(); + let lockfile = Lockfile::from_graph(root.clone(), &graph1); + + // Create a new graph with a dependency + let mut graph2 = DependencyGraph::new(); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let alias: DepAlias = "eth".parse().expect("valid alias"); + + graph2.direct.insert(alias, hash.clone()); + graph2.nodes.insert( + hash.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash.clone(), create_test_manifest()), + deps: vec![], + }, + ); + + //* When + let result = lockfile.verify(&graph2); + + //* Then + assert!(result.is_err(), "verification should fail"); + let err = result.expect_err("should have error"); + assert!( + matches!(err, LockfileError::DirectDepsMismatch { .. }), + "should be DirectDepsMismatch error, got: {:?}", + err + ); + } + + #[test] + fn verify_fails_for_extra_node_in_lockfile() { + //* Given + let root = create_root_info(); + let mut graph = DependencyGraph::new(); + + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let alias: DepAlias = "eth".parse().expect("valid alias"); + + graph.direct.insert(alias, hash.clone()); + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash.clone(), create_test_manifest()), + deps: vec![], + }, + ); + + let lockfile = Lockfile::from_graph(root, &graph); + + // Remove the node from the graph + graph.nodes.clear(); + graph.direct.clear(); + + //* When + let result = lockfile.verify(&graph); + + //* Then + assert!(result.is_err(), "verification should fail"); + let err = result.expect_err("should have error"); + assert!( + matches!(err, LockfileError::DirectDepsMismatch { .. }), + "should be DirectDepsMismatch error for direct deps mismatch, got: {:?}", + err + ); + } + + #[test] + fn file_roundtrip_succeeds() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let lockfile_path = temp_dir.path().join("amp.lock"); + + let root = create_root_info(); + let mut graph = DependencyGraph::new(); + + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let alias: DepAlias = "eth".parse().expect("valid alias"); + + graph.direct.insert(alias, hash.clone()); + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash.clone(), create_test_manifest()), + deps: vec![], + }, + ); + + let original = Lockfile::from_graph(root, &graph); + + //* When + original + .write(&lockfile_path) + .expect("write should succeed"); + let loaded = Lockfile::read(&lockfile_path); + + //* Then + assert!(loaded.is_ok(), "read should succeed"); + let loaded = loaded.expect("should have lockfile"); + assert_eq!(loaded, original); + } + + #[test] + fn read_nonexistent_file_returns_error() { + //* Given + let path = Path::new("/nonexistent/path/amp.lock"); + + //* When + let result = Lockfile::read(path); + + //* Then + assert!(result.is_err(), "read should fail for nonexistent file"); + let err = result.expect_err("should have error"); + assert!( + matches!(err, LockfileError::ReadFile { .. }), + "should be ReadFile error" + ); + } + + #[test] + fn path_in_returns_correct_path() { + //* Given + let dir = Path::new("/some/project"); + + //* When + let path = Lockfile::path_in(dir); + + //* Then + assert_eq!(path, PathBuf::from("/some/project/amp.lock")); + } + + #[test] + fn lockfile_nodes_have_namespace_name() { + //* Given + let root = create_root_info(); + let mut graph = DependencyGraph::new(); + + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let alias: DepAlias = "eth".parse().expect("valid alias"); + + // Create resolved dependency with specific namespace/name + let resolved = ResolvedDependency { + hash: hash.clone(), + manifest: create_test_manifest(), + namespace: "graph".parse().expect("valid namespace"), + name: "eth_mainnet".parse().expect("valid name"), + }; + + graph.direct.insert(alias, hash.clone()); + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved, + deps: vec![], + }, + ); + + //* When + let lockfile = Lockfile::from_graph(root, &graph); + + //* Then + let node = &lockfile.nodes[&hash]; + assert_eq!(node.namespace, "graph", "namespace should be captured"); + assert_eq!(node.name, "eth_mainnet", "name should be captured"); + } + + #[test] + fn verify_with_root_succeeds_for_matching_root() { + //* Given + let root = create_root_info(); + let graph = DependencyGraph::new(); + let lockfile = Lockfile::from_graph(root.clone(), &graph); + + //* When + let result = lockfile.verify_with_root(Some(&root), &graph); + + //* Then + assert!( + result.is_ok(), + "verification should succeed for matching root" + ); + } + + #[test] + fn verify_with_root_fails_for_mismatched_namespace() { + //* Given + let root = create_root_info(); + let graph = DependencyGraph::new(); + let lockfile = Lockfile::from_graph(root, &graph); + + let different_root = RootInfo { + namespace: "different_namespace".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + version: "1.0.0".parse().expect("valid version"), + }; + + //* When + let result = lockfile.verify_with_root(Some(&different_root), &graph); + + //* Then + assert!( + result.is_err(), + "verification should fail for mismatched namespace" + ); + let err = result.expect_err("should have error"); + assert!( + matches!(err, LockfileError::RootMismatch { .. }), + "should be RootMismatch error, got: {:?}", + err + ); + } + + #[test] + fn verify_with_root_fails_for_mismatched_version() { + //* Given + let root = create_root_info(); + let graph = DependencyGraph::new(); + let lockfile = Lockfile::from_graph(root, &graph); + + let different_root = RootInfo { + namespace: "test_namespace".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + version: "2.0.0".parse().expect("valid version"), + }; + + //* When + let result = lockfile.verify_with_root(Some(&different_root), &graph); + + //* Then + assert!( + result.is_err(), + "verification should fail for mismatched version" + ); + let err = result.expect_err("should have error"); + assert!( + matches!(err, LockfileError::RootMismatch { .. }), + "should be RootMismatch error, got: {:?}", + err + ); + } + + #[test] + fn verify_dependencies_succeeds_for_matching_config() { + //* Given + let root = create_root_info(); + let mut graph = DependencyGraph::new(); + + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let alias: DepAlias = "eth".parse().expect("valid alias"); + + graph.direct.insert(alias.clone(), hash.clone()); + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash.clone(), create_test_manifest()), + deps: vec![], + }, + ); + + let lockfile = Lockfile::from_graph(root, &graph); + + let reference: DepReference = "test/dataset@1.0.0".parse().expect("valid reference"); + let mut dependencies = BTreeMap::new(); + dependencies.insert(alias, reference); + + //* When + let result = lockfile.verify_dependencies(&dependencies); + + //* Then + assert!(result.is_ok(), "verification should succeed"); + } + + #[test] + fn verify_dependencies_fails_for_hash_mismatch() { + //* Given + let root = create_root_info(); + let mut graph = DependencyGraph::new(); + + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let alias: DepAlias = "eth".parse().expect("valid alias"); + + graph.direct.insert(alias.clone(), hash.clone()); + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash.clone(), create_test_manifest()), + deps: vec![], + }, + ); + + let lockfile = Lockfile::from_graph(root, &graph); + + let other_hash: Hash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .parse() + .expect("valid hash"); + let reference: DepReference = format!("test/dataset@{other_hash}") + .parse() + .expect("valid reference"); + let mut dependencies = BTreeMap::new(); + dependencies.insert(alias, reference); + + //* When + let result = lockfile.verify_dependencies(&dependencies); + + //* Then + assert!(result.is_err(), "verification should fail"); + let err = result.expect_err("should have error"); + assert!( + matches!(err, LockfileError::DirectDepHashMismatch { .. }), + "should be DirectDepHashMismatch error, got: {:?}", + err + ); + } + + #[test] + fn verify_dependencies_fails_for_fqn_mismatch() { + //* Given + let root = create_root_info(); + let mut graph = DependencyGraph::new(); + + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let alias: DepAlias = "eth".parse().expect("valid alias"); + + graph.direct.insert(alias.clone(), hash.clone()); + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash.clone(), create_test_manifest()), + deps: vec![], + }, + ); + + let lockfile = Lockfile::from_graph(root, &graph); + + let reference: DepReference = "other/dataset@1.0.0".parse().expect("valid reference"); + let mut dependencies = BTreeMap::new(); + dependencies.insert(alias, reference); + + //* When + let result = lockfile.verify_dependencies(&dependencies); + + //* Then + assert!(result.is_err(), "verification should fail"); + let err = result.expect_err("should have error"); + assert!( + matches!(err, LockfileError::DirectDepFqnMismatch { .. }), + "should be DirectDepFqnMismatch error, got: {:?}", + err + ); + } +} diff --git a/crates/core/dataset-authoring/src/manifest.rs b/crates/core/dataset-authoring/src/manifest.rs new file mode 100644 index 000000000..73f759142 --- /dev/null +++ b/crates/core/dataset-authoring/src/manifest.rs @@ -0,0 +1,1139 @@ +//! Authoring manifest generation. +//! +//! Generates the canonical `manifest.json` with file references and content hashes. +//! This is distinct from the runtime manifest in `datasets-derived` which uses +//! inline SQL and schemas. +//! +//! # Manifest Structure +//! +//! The authoring manifest uses file references instead of inline content: +//! +//! ```json +//! { +//! "namespace": "my_namespace", +//! "name": "my_dataset", +//! "kind": "derived", +//! "dependencies": { +//! "eth": { +//! "hash": "abc123..." +//! } +//! }, +//! "tables": { +//! "transfers": { +//! "sql": { "path": "tables/transfers.sql", "hash": "..." }, +//! "ipc": { "path": "tables/transfers.ipc", "hash": "..." }, +//! "network": "mainnet" +//! } +//! }, +//! "functions": { +//! "decode": { +//! "inputTypes": ["Binary"], +//! "outputType": "Utf8", +//! "source": { "path": "functions/decode.js", "hash": "..." } +//! } +//! } +//! } +//! ``` +//! +//! For raw tables (no SQL definition), the `sql` field is omitted. +//! +//! # Identity Hash +//! +//! The manifest identity is computed as `sha256()` using +//! RFC 8785 (JCS) canonicalization via `serde_json_canonicalizer`. + +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, +}; + +use datasets_common::{ + hash::Hash, manifest::DataType, name::Name, namespace::Namespace, network_id::NetworkId, + table_name::TableName, +}; +use datasets_derived::{deps::DepAlias, func_name::FuncName}; +use serde::{Deserialize, Serialize}; + +use crate::{ + canonical::{CanonicalizeError, canonicalize, hash_canonical}, + config::AmpYaml, + files::{FileRef, FileRefError}, + resolver::DependencyGraph, +}; + +/// Errors that occur during manifest generation. +#[derive(Debug, thiserror::Error)] +pub enum ManifestError { + /// Failed to create file reference for SQL file. + /// + /// This occurs when the SQL file cannot be read or hashed. + #[error("failed to create file reference for SQL file '{table_name}'")] + SqlFileRef { + /// The table name associated with the SQL file. + table_name: TableName, + /// The underlying file reference error. + #[source] + source: FileRefError, + }, + + /// Failed to create file reference for IPC schema file. + /// + /// This occurs when the IPC schema file cannot be read or hashed. + #[error("failed to create file reference for IPC schema file '{table_name}'")] + IpcFileRef { + /// The table name associated with the IPC schema file. + table_name: TableName, + /// The underlying file reference error. + #[source] + source: FileRefError, + }, + + /// Failed to create file reference for function source file. + /// + /// This occurs when the function source file cannot be read or hashed. + #[error("failed to create file reference for function '{func_name}'")] + FunctionFileRef { + /// The function name. + func_name: FuncName, + /// The underlying file reference error. + #[source] + source: FileRefError, + }, + + /// Missing dependency in resolved graph. + /// + /// This occurs when a dependency alias from amp.yaml is not found + /// in the resolved dependency graph. + #[error("dependency '{alias}' not found in resolved dependency graph")] + MissingDependency { + /// The missing dependency alias. + alias: DepAlias, + }, + + /// Failed to infer network for table. + /// + /// This occurs when a table has no dependencies to infer network from, + /// or when dependencies have conflicting networks. + #[error("cannot infer network for table '{table_name}': {reason}")] + NetworkInference { + /// The table name. + table_name: TableName, + /// Reason for the failure. + reason: String, + }, + + /// Failed to parse data type for function. + /// + /// This occurs when a function's input or output type string + /// cannot be parsed as a valid Arrow data type. + #[error("invalid data type '{type_str}' for function '{func_name}'")] + InvalidDataType { + /// The function name. + func_name: FuncName, + /// The invalid type string. + type_str: String, + /// The underlying parse error. + #[source] + source: serde_json::Error, + }, + + /// Failed to canonicalize manifest for hashing. + /// + /// This occurs when the manifest cannot be converted to canonical JSON. + #[error("failed to canonicalize manifest")] + Canonicalize(#[source] CanonicalizeError), + + /// Failed to serialize manifest to JSON. + /// + /// This occurs when the manifest struct cannot be converted to JSON. + #[error("failed to serialize manifest to JSON")] + Serialize(#[source] serde_json::Error), +} + +/// Dependency information in the authoring manifest. +/// +/// Contains the resolved hash and the original reference information +/// needed for legacy manifest conversion. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DepInfo { + /// The content hash of the resolved dependency manifest. + pub hash: Hash, + /// The original namespace of the dependency. + pub namespace: Namespace, + /// The original name of the dependency. + pub name: Name, +} + +/// Table definition in the authoring manifest. +/// +/// References SQL and IPC schema files by path and content hash. +/// For derived tables, `sql` contains the CTAS query. For raw tables, +/// `sql` is `None` since raw tables have no SQL definition. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TableDef { + /// Reference to the SQL file containing the CTAS statement. + /// `None` for raw tables which have no SQL definition. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sql: Option, + /// Reference to the Arrow IPC schema file. + pub ipc: FileRef, + /// Network this table belongs to (inferred from dependencies). + pub network: NetworkId, +} + +/// Function definition in the authoring manifest. +/// +/// Contains the function signature and source file reference. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AuthoringFunctionDef { + /// Arrow data types for function input parameters. + pub input_types: Vec, + /// Arrow data type for function return value. + pub output_type: DataType, + /// Reference to the function source file. + pub source: FileRef, +} + +/// The authoring manifest for a derived dataset. +/// +/// This manifest uses file references instead of inline content, +/// which is the format used during the authoring workflow. It is +/// converted to the runtime manifest format (with inline content) +/// via the legacy bridge for deployment. +/// +/// # Identity +/// +/// The manifest identity is computed as `sha256()` +/// using JCS canonicalization. This hash uniquely identifies the manifest +/// content and is used for content-addressable storage. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuthoringManifest { + /// Dataset namespace. + pub namespace: Namespace, + /// Dataset name. + pub name: Name, + /// Dataset kind (always "derived" for authoring). + pub kind: String, + /// Resolved dependencies with their content hashes. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub dependencies: BTreeMap, + /// Table definitions with file references. + pub tables: BTreeMap, + /// Function definitions with file references. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub functions: BTreeMap, +} + +impl AuthoringManifest { + /// Computes the canonical JSON representation of the manifest. + /// + /// Uses RFC 8785 (JCS) canonicalization via `serde_json_canonicalizer` to + /// produce deterministic JSON output with sorted keys and no whitespace. + /// + /// # Errors + /// + /// Returns [`ManifestError::Serialize`] if the manifest cannot be serialized. + /// Returns [`ManifestError::Canonicalize`] if canonicalization fails. + pub fn canonical_json(&self) -> Result { + let value = serde_json::to_value(self).map_err(ManifestError::Serialize)?; + canonicalize(&value).map_err(ManifestError::Canonicalize) + } + + /// Computes the identity hash of the manifest. + /// + /// The identity is `sha256()`. + /// + /// # Errors + /// + /// Returns [`ManifestError::Serialize`] if the manifest cannot be serialized. + /// Returns [`ManifestError::Canonicalize`] if canonicalization fails. + pub fn identity(&self) -> Result { + let value = serde_json::to_value(self).map_err(ManifestError::Serialize)?; + hash_canonical(&value).map_err(ManifestError::Canonicalize) + } + + /// Serializes the manifest to pretty-printed JSON. + /// + /// This is useful for writing the manifest to a file for human inspection. + /// For identity computation, use [`canonical_json`](Self::canonical_json) instead. + /// + /// # Errors + /// + /// Returns [`ManifestError::Serialize`] if serialization fails. + pub fn to_json_pretty(&self) -> Result { + serde_json::to_string_pretty(self).map_err(ManifestError::Serialize) + } +} + +/// Builder for constructing an [`AuthoringManifest`] from authoring inputs. +/// +/// This builder takes: +/// - The parsed `amp.yaml` configuration +/// - The resolved dependency graph +/// - The base path for file reference normalization +/// - The directory containing rendered SQL and IPC schema files +/// - The discovered tables (from model discovery) +/// - A network resolver for inferring table networks +/// +/// # Example +/// +/// ```ignore +/// use dataset_authoring::manifest::ManifestBuilder; +/// +/// let manifest = ManifestBuilder::new(&config, &dep_graph, project_dir, &tables_dir, &tables) +/// .with_network_resolver(|table, deps| infer_network(table, deps)) +/// .build()?; +/// ``` +pub struct ManifestBuilder<'a, F> { + config: &'a AmpYaml, + dep_graph: &'a DependencyGraph, + base_path: &'a Path, + tables_dir: &'a Path, + /// Discovered tables (table name -> SQL file path relative to base_path). + /// This replaces the old `config.tables` field. + tables: &'a BTreeMap, + network_resolver: F, +} + +impl<'a> ManifestBuilder<'a, fn(&TableName, &DependencyGraph) -> Result> { + /// Creates a new manifest builder with the default network resolver. + /// + /// The default network resolver requires all dependencies to have the + /// same network and uses that network for all tables. + /// + /// # Arguments + /// + /// * `config` - The parsed amp.yaml configuration + /// * `dep_graph` - The resolved dependency graph + /// * `base_path` - Base path for file reference normalization + /// * `tables_dir` - Directory containing rendered SQL and IPC schema files + /// * `tables` - Discovered tables (table name -> SQL file path relative to base_path) + pub fn new( + config: &'a AmpYaml, + dep_graph: &'a DependencyGraph, + base_path: &'a Path, + tables_dir: &'a Path, + tables: &'a BTreeMap, + ) -> Self { + Self { + config, + dep_graph, + base_path, + tables_dir, + tables, + network_resolver: default_network_resolver, + } + } +} + +impl<'a, F> ManifestBuilder<'a, F> +where + F: Fn(&TableName, &DependencyGraph) -> Result, +{ + /// Sets a custom network resolver. + /// + /// The resolver is called for each table to determine its network. + pub fn with_network_resolver(self, resolver: G) -> ManifestBuilder<'a, G> + where + G: Fn(&TableName, &DependencyGraph) -> Result, + { + ManifestBuilder { + config: self.config, + dep_graph: self.dep_graph, + base_path: self.base_path, + tables_dir: self.tables_dir, + tables: self.tables, + network_resolver: resolver, + } + } + + /// Builds the authoring manifest. + /// + /// This method: + /// 1. Maps dependencies from amp.yaml to resolved hashes + /// 2. Creates file references for SQL and schema files + /// 3. Creates file references for function source files + /// 4. Infers networks for tables using the network resolver + /// + /// # Errors + /// + /// Returns [`ManifestError`] if any file references cannot be created + /// or if network inference fails. + pub fn build(self) -> Result { + // Build dependencies map + let dependencies = self.build_dependencies()?; + + // Build tables map + let tables = self.build_tables()?; + + // Build functions map + let functions = self.build_functions()?; + + Ok(AuthoringManifest { + namespace: self.config.namespace.clone(), + name: self.config.name.clone(), + kind: "derived".to_string(), + dependencies, + tables, + functions, + }) + } + + fn build_dependencies(&self) -> Result, ManifestError> { + let mut dependencies = BTreeMap::new(); + + for (alias, dep_ref) in &self.config.dependencies { + let hash = self.dep_graph.direct.get(alias).ok_or_else(|| { + ManifestError::MissingDependency { + alias: alias.clone(), + } + })?; + + let fqn = dep_ref.fqn(); + dependencies.insert( + alias.clone(), + DepInfo { + hash: hash.clone(), + namespace: fqn.namespace().clone(), + name: fqn.name().clone(), + }, + ); + } + + Ok(dependencies) + } + + fn build_tables(&self) -> Result, ManifestError> { + let mut tables = BTreeMap::new(); + + for table_name in self.tables.keys() { + // SQL file path: tables/.sql (rendered output) + let sql_file_path = self.tables_dir.join(format!("{}.sql", table_name)); + let sql_ref = FileRef::from_file(&sql_file_path, self.base_path).map_err(|err| { + ManifestError::SqlFileRef { + table_name: table_name.clone(), + source: err, + } + })?; + + // IPC schema file path: tables/.ipc + let ipc_file_path = self.tables_dir.join(format!("{}.ipc", table_name)); + let ipc_ref = FileRef::from_file(&ipc_file_path, self.base_path).map_err(|err| { + ManifestError::IpcFileRef { + table_name: table_name.clone(), + source: err, + } + })?; + + // Infer network using the resolver + let network = (self.network_resolver)(table_name, self.dep_graph)?; + + tables.insert( + table_name.clone(), + TableDef { + sql: Some(sql_ref), + ipc: ipc_ref, + network, + }, + ); + } + + Ok(tables) + } + + fn build_functions(&self) -> Result, ManifestError> { + let mut functions = BTreeMap::new(); + + let Some(config_functions) = &self.config.functions else { + return Ok(functions); + }; + + for (func_name, func_def) in config_functions { + // Use canonical output path: functions/.js + // build.rs copies input files to this canonical location + let canonical_path = format!("functions/{}.js", func_name); + let source_path = self.base_path.join(&canonical_path); + let source_ref = FileRef::from_file(&source_path, self.base_path).map_err(|err| { + ManifestError::FunctionFileRef { + func_name: func_name.clone(), + source: err, + } + })?; + + let input_types = parse_data_types(&func_def.input_types, func_name)?; + let output_type = parse_data_type(&func_def.output_type, func_name)?; + + functions.insert( + func_name.clone(), + AuthoringFunctionDef { + input_types, + output_type, + source: source_ref, + }, + ); + } + + Ok(functions) + } +} + +/// Parses a list of data type strings into Arrow data types. +fn parse_data_types( + type_strs: &[String], + func_name: &FuncName, +) -> Result, ManifestError> { + type_strs + .iter() + .map(|s| parse_data_type(s, func_name)) + .collect() +} + +/// Parses a data type string into an Arrow data type. +/// +/// Uses JSON parsing with the DataType wrapper from datasets-common. +fn parse_data_type(type_str: &str, func_name: &FuncName) -> Result { + // DataType serializes as a string (transparent), so we can parse it directly + serde_json::from_value(serde_json::Value::String(type_str.to_string())).map_err(|err| { + ManifestError::InvalidDataType { + func_name: func_name.clone(), + type_str: type_str.to_string(), + source: err, + } + }) +} + +/// Default network resolver that infers network from dependencies. +/// +/// This resolver: +/// 1. Collects all unique networks from dependencies +/// 2. If all dependencies have the same network, uses that network +/// 3. If there are no dependencies, returns an error +/// 4. If dependencies have different networks, returns an error +pub(crate) fn default_network_resolver( + table_name: &TableName, + dep_graph: &DependencyGraph, +) -> Result { + if dep_graph.is_empty() { + return Err(ManifestError::NetworkInference { + table_name: table_name.clone(), + reason: "no dependencies to infer network from".to_string(), + }); + } + + // Collect all unique networks from dependencies + let mut networks: Vec = Vec::new(); + for node in dep_graph.nodes.values() { + for table in node.resolved.manifest.tables.values() { + if !networks.contains(&table.network) { + networks.push(table.network.clone()); + } + } + } + + match networks.len() { + 0 => Err(ManifestError::NetworkInference { + table_name: table_name.clone(), + reason: "dependencies have no tables with network information".to_string(), + }), + 1 => Ok(networks.into_iter().next().expect("checked len == 1")), + _ => Err(ManifestError::NetworkInference { + table_name: table_name.clone(), + reason: format!( + "dependencies have conflicting networks: {}", + networks + .iter() + .map(|n| n.to_string()) + .collect::>() + .join(", ") + ), + }), + } +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, fs}; + + use datasets_common::dataset_kind_str::DatasetKindStr; + use tempfile::TempDir; + + use super::*; + use crate::{ + dependency_manifest::{DependencyManifest, DependencyTable}, + resolver::{DependencyNode, ResolvedDependency}, + }; + + // ============================================================ + // AuthoringManifest tests + // ============================================================ + + fn create_test_manifest() -> AuthoringManifest { + AuthoringManifest { + namespace: "test_ns".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + kind: "derived".to_string(), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + } + } + + #[test] + fn authoring_manifest_serializes_to_json() { + //* Given + let manifest = create_test_manifest(); + + //* When + let result = manifest.to_json_pretty(); + + //* Then + let json = result.expect("serialization should succeed"); + assert!(json.contains("\"namespace\": \"test_ns\"")); + assert!(json.contains("\"name\": \"test_dataset\"")); + assert!(json.contains("\"kind\": \"derived\"")); + } + + #[test] + fn authoring_manifest_canonical_json_is_deterministic() { + //* Given + let manifest = create_test_manifest(); + + //* When + let json1 = manifest.canonical_json(); + let json2 = manifest.canonical_json(); + + //* Then + let j1 = json1.expect("should succeed"); + let j2 = json2.expect("should succeed"); + assert_eq!(j1, j2, "canonical JSON should be deterministic"); + } + + #[test] + fn authoring_manifest_canonical_json_has_no_whitespace() { + //* Given + let manifest = create_test_manifest(); + + //* When + let result = manifest.canonical_json(); + + //* Then + let json = result.expect("should succeed"); + assert!(!json.contains('\n'), "should have no newlines"); + // Note: spaces may exist in string values, but not between tokens + assert!(!json.contains(": "), "should have no space after colons"); + } + + #[test] + fn authoring_manifest_identity_is_deterministic() { + //* Given + let manifest = create_test_manifest(); + + //* When + let hash1 = manifest.identity(); + let hash2 = manifest.identity(); + + //* Then + let h1 = hash1.expect("should succeed"); + let h2 = hash2.expect("should succeed"); + assert_eq!(h1, h2, "identity hash should be deterministic"); + } + + #[test] + fn authoring_manifest_identity_differs_for_different_content() { + //* Given + let manifest1 = create_test_manifest(); + let mut manifest2 = create_test_manifest(); + manifest2.name = "other_dataset".parse().expect("valid name"); + + //* When + let hash1 = manifest1.identity(); + let hash2 = manifest2.identity(); + + //* Then + let h1 = hash1.expect("should succeed"); + let h2 = hash2.expect("should succeed"); + assert_ne!( + h1, h2, + "different content should produce different identity" + ); + } + + #[test] + fn authoring_manifest_roundtrip_serialization() { + //* Given + let original = create_test_manifest(); + + //* When + let json = serde_json::to_string(&original).expect("serialize should succeed"); + let deserialized: AuthoringManifest = + serde_json::from_str(&json).expect("deserialize should succeed"); + + //* Then + assert_eq!(deserialized, original); + } + + #[test] + fn authoring_manifest_skips_empty_optional_fields() { + //* Given + let manifest = AuthoringManifest { + namespace: "test_ns".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + kind: "derived".to_string(), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + }; + + //* When + let result = manifest.to_json_pretty(); + + //* Then + let json = result.expect("serialization should succeed"); + assert!( + !json.contains("dependencies"), + "should skip empty dependencies" + ); + assert!(!json.contains("functions"), "should skip empty functions"); + } + + // ============================================================ + // DepInfo tests + // ============================================================ + + #[test] + fn dep_info_serializes_correctly() { + //* Given + let dep_info = DepInfo { + hash: "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"), + namespace: "test_ns".parse().expect("valid namespace"), + name: "test_dep".parse().expect("valid name"), + }; + + //* When + let json = serde_json::to_string(&dep_info); + + //* Then + let json_str = json.expect("serialization should succeed"); + assert!( + json_str.contains("b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9") + ); + assert!(json_str.contains("test_ns")); + assert!(json_str.contains("test_dep")); + } + + // ============================================================ + // TableDef tests + // ============================================================ + + #[test] + fn table_def_serializes_correctly() { + //* Given + let table_def = TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .parse() + .expect("valid hash"), + )), + ipc: FileRef::new( + "tables/transfers.ipc".to_string(), + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + .parse() + .expect("valid hash"), + ), + network: "mainnet".parse().expect("valid network"), + }; + + //* When + let json = serde_json::to_string_pretty(&table_def); + + //* Then + let json_str = json.expect("serialization should succeed"); + assert!(json_str.contains("tables/transfers.sql")); + assert!(json_str.contains("tables/transfers.ipc")); + assert!(json_str.contains("mainnet")); + } + + #[test] + fn table_def_without_sql_omits_sql_field() { + //* Given - a raw table with no SQL definition + let table_def = TableDef { + sql: None, + ipc: FileRef::new( + "tables/raw_events.ipc".to_string(), + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + .parse() + .expect("valid hash"), + ), + network: "mainnet".parse().expect("valid network"), + }; + + //* When + let json = serde_json::to_string_pretty(&table_def); + + //* Then + let json_str = json.expect("serialization should succeed"); + assert!( + !json_str.contains("\"sql\""), + "sql field should be omitted when None" + ); + assert!(json_str.contains("tables/raw_events.ipc")); + assert!(json_str.contains("mainnet")); + } + + #[test] + fn table_def_roundtrip_with_optional_sql() { + //* Given - a table with SQL + let table_def_with_sql = TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .parse() + .expect("valid hash"), + )), + ipc: FileRef::new( + "tables/transfers.ipc".to_string(), + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + .parse() + .expect("valid hash"), + ), + network: "mainnet".parse().expect("valid network"), + }; + + //* When - roundtrip serialize/deserialize + let json = serde_json::to_string(&table_def_with_sql).expect("serialize should succeed"); + let deserialized: TableDef = + serde_json::from_str(&json).expect("deserialize should succeed"); + + //* Then + assert_eq!(deserialized, table_def_with_sql); + + //* Given - a table without SQL (raw table) + let table_def_without_sql = TableDef { + sql: None, + ipc: FileRef::new( + "tables/raw_events.ipc".to_string(), + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + .parse() + .expect("valid hash"), + ), + network: "mainnet".parse().expect("valid network"), + }; + + //* When + let json = serde_json::to_string(&table_def_without_sql).expect("serialize should succeed"); + let deserialized: TableDef = + serde_json::from_str(&json).expect("deserialize should succeed"); + + //* Then + assert_eq!(deserialized, table_def_without_sql); + } + + // ============================================================ + // ManifestBuilder tests + // ============================================================ + + fn create_test_config() -> AmpYaml { + let yaml = r#" +amp: 1.0.0 +namespace: test_ns +name: test_dataset +version: 1.0.0 +tables: tables +"#; + AmpYaml::from_yaml(yaml).expect("valid config") + } + + fn create_test_tables() -> BTreeMap { + let mut tables = BTreeMap::new(); + tables.insert( + "transfers".parse().expect("valid table name"), + PathBuf::from("tables/transfers.sql"), + ); + tables + } + + fn create_test_dep_graph_with_network(network: &str) -> DependencyGraph { + let mut graph = DependencyGraph::new(); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + let mut tables = BTreeMap::new(); + tables.insert( + "source_table".parse().expect("valid table name"), + DependencyTable { + schema: datasets_common::manifest::TableSchema { + arrow: datasets_common::manifest::ArrowSchema { fields: vec![] }, + }, + network: network.parse().expect("valid network"), + }, + ); + + let manifest = DependencyManifest { + kind: DatasetKindStr::new("manifest".to_string()), + dependencies: BTreeMap::new(), + tables, + functions: BTreeMap::new(), + }; + + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved: ResolvedDependency { + hash: hash.clone(), + manifest, + namespace: "test".parse().expect("valid namespace"), + name: "dependency".parse().expect("valid name"), + }, + deps: vec![], + }, + ); + + graph + } + + #[test] + fn manifest_builder_builds_minimal_manifest() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let tables_dir = dir.path().join("tables"); + fs::create_dir(&tables_dir).expect("should create tables dir"); + + // Create SQL and IPC schema files + fs::write(tables_dir.join("transfers.sql"), "SELECT 1").expect("should write sql"); + fs::write(tables_dir.join("transfers.ipc"), b"dummy ipc").expect("should write ipc"); + + let config = create_test_config(); + let tables = create_test_tables(); + let dep_graph = create_test_dep_graph_with_network("mainnet"); + + //* When + let result = + ManifestBuilder::new(&config, &dep_graph, dir.path(), &tables_dir, &tables).build(); + + //* Then + let manifest = result.expect("build should succeed"); + assert_eq!(manifest.namespace, "test_ns"); + assert_eq!(manifest.name, "test_dataset"); + assert_eq!(manifest.kind, "derived"); + assert_eq!(manifest.tables.len(), 1); + assert!(manifest.tables.contains_key(&"transfers".parse().unwrap())); + } + + #[test] + fn manifest_builder_infers_network_from_deps() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let tables_dir = dir.path().join("tables"); + fs::create_dir(&tables_dir).expect("should create tables dir"); + + fs::write(tables_dir.join("transfers.sql"), "SELECT 1").expect("should write sql"); + fs::write(tables_dir.join("transfers.ipc"), b"dummy ipc").expect("should write ipc"); + + let config = create_test_config(); + let tables = create_test_tables(); + let dep_graph = create_test_dep_graph_with_network("mainnet"); + + //* When + let result = + ManifestBuilder::new(&config, &dep_graph, dir.path(), &tables_dir, &tables).build(); + + //* Then + let manifest = result.expect("build should succeed"); + let table = manifest + .tables + .get(&"transfers".parse().unwrap()) + .expect("should have table"); + assert_eq!(table.network.to_string(), "mainnet"); + } + + #[test] + fn manifest_builder_fails_on_missing_sql_file() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let tables_dir = dir.path().join("tables"); + fs::create_dir(&tables_dir).expect("should create tables dir"); + // Don't create the SQL file + + let config = create_test_config(); + let tables = create_test_tables(); + let dep_graph = create_test_dep_graph_with_network("mainnet"); + + //* When + let result = + ManifestBuilder::new(&config, &dep_graph, dir.path(), &tables_dir, &tables).build(); + + //* Then + assert!(result.is_err()); + let err = result.expect_err("should fail"); + assert!( + matches!(err, ManifestError::SqlFileRef { .. }), + "should be SqlFileRef error, got: {:?}", + err + ); + } + + #[test] + fn manifest_builder_fails_on_missing_ipc_file() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let tables_dir = dir.path().join("tables"); + fs::create_dir(&tables_dir).expect("should create tables dir"); + + // Create SQL but not IPC schema + fs::write(tables_dir.join("transfers.sql"), "SELECT 1").expect("should write sql"); + + let config = create_test_config(); + let tables = create_test_tables(); + let dep_graph = create_test_dep_graph_with_network("mainnet"); + + //* When + let result = + ManifestBuilder::new(&config, &dep_graph, dir.path(), &tables_dir, &tables).build(); + + //* Then + assert!(result.is_err()); + let err = result.expect_err("should fail"); + assert!( + matches!(err, ManifestError::IpcFileRef { .. }), + "should be IpcFileRef error, got: {:?}", + err + ); + } + + #[test] + fn manifest_builder_with_custom_network_resolver() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let tables_dir = dir.path().join("tables"); + fs::create_dir(&tables_dir).expect("should create tables dir"); + + fs::write(tables_dir.join("transfers.sql"), "SELECT 1").expect("should write sql"); + fs::write(tables_dir.join("transfers.ipc"), b"dummy ipc").expect("should write ipc"); + + let config = create_test_config(); + let tables = create_test_tables(); + let dep_graph = DependencyGraph::new(); // Empty graph + + // Custom resolver that always returns "testnet" + let custom_resolver = + |_table: &TableName, _graph: &DependencyGraph| -> Result { + Ok("testnet".parse().expect("valid network")) + }; + + //* When + let result = ManifestBuilder::new(&config, &dep_graph, dir.path(), &tables_dir, &tables) + .with_network_resolver(custom_resolver) + .build(); + + //* Then + let manifest = result.expect("build should succeed"); + let table = manifest + .tables + .get(&"transfers".parse().unwrap()) + .expect("should have table"); + assert_eq!(table.network.to_string(), "testnet"); + } + + // ============================================================ + // Network inference tests + // ============================================================ + + #[test] + fn default_network_resolver_fails_on_empty_graph() { + //* Given + let dep_graph = DependencyGraph::new(); + let table_name: TableName = "test".parse().expect("valid"); + + //* When + let result = default_network_resolver(&table_name, &dep_graph); + + //* Then + assert!(result.is_err()); + let err = result.expect_err("should fail"); + assert!( + matches!(err, ManifestError::NetworkInference { .. }), + "should be NetworkInference error, got: {:?}", + err + ); + } + + #[test] + fn default_network_resolver_succeeds_with_single_network() { + //* Given + let dep_graph = create_test_dep_graph_with_network("mainnet"); + let table_name: TableName = "test".parse().expect("valid"); + + //* When + let result = default_network_resolver(&table_name, &dep_graph); + + //* Then + let network = result.expect("should succeed"); + assert_eq!(network.to_string(), "mainnet"); + } + + // ============================================================ + // Function path canonicalization tests + // ============================================================ + + #[test] + fn manifest_builder_uses_canonical_function_path() { + //* Given + // Create a config with a non-canonical function path (lib/helpers/my_func.js) + let yaml = r#" +amp: 1.0.0 +namespace: test_ns +name: test_dataset +version: 1.0.0 +tables: tables +functions: + my_func: + source: lib/helpers/my_func.js + input_types: ["Binary"] + output_type: Utf8 +"#; + let config = AmpYaml::from_yaml(yaml).expect("valid config"); + let tables = create_test_tables(); + + // Set up directory with canonical output structure + let dir = TempDir::new().expect("should create temp dir"); + let tables_dir = dir.path().join("tables"); + let functions_dir = dir.path().join("functions"); + fs::create_dir_all(&tables_dir).expect("should create tables dir"); + fs::create_dir_all(&functions_dir).expect("should create functions dir"); + + // Create required files at canonical locations + fs::write(tables_dir.join("transfers.sql"), "SELECT 1").expect("should write sql"); + fs::write(tables_dir.join("transfers.ipc"), b"dummy ipc").expect("should write ipc"); + + // Function file at CANONICAL path (build.rs copies to this location) + fs::write( + functions_dir.join("my_func.js"), + "function my_func(input) { return input; }", + ) + .expect("should write function"); + + let dep_graph = create_test_dep_graph_with_network("mainnet"); + + //* When + let result = + ManifestBuilder::new(&config, &dep_graph, dir.path(), &tables_dir, &tables).build(); + + //* Then + let manifest = result.expect("build should succeed"); + let func = manifest + .functions + .get(&"my_func".parse().unwrap()) + .expect("should have function"); + + // Verify the manifest uses the canonical path, not the input path + assert_eq!( + func.source.path, "functions/my_func.js", + "function path should be canonical (functions/.js), not input path" + ); + } +} diff --git a/crates/core/dataset-authoring/src/package.rs b/crates/core/dataset-authoring/src/package.rs new file mode 100644 index 000000000..81cb12a6e --- /dev/null +++ b/crates/core/dataset-authoring/src/package.rs @@ -0,0 +1,754 @@ +//! Package assembly for dataset authoring. +//! +//! Creates deterministic `dataset.tgz` archives containing: +//! - `manifest.json` (canonical) +//! - `tables/
.sql` (rendered, post-Jinja) for derived tables +//! - `tables/
.ipc` (Arrow IPC schema) +//! - `functions/.js` (if any) +//! +//! # Determinism +//! +//! Archives are deterministic for identical inputs: +//! - Entries sorted alphabetically by path +//! - mtime set to Unix epoch (0) +//! - Consistent permissions (0o644 for files) +//! - No ownership metadata (uid/gid = 0) +//! +//! # Example +//! +//! ```ignore +//! use std::path::Path; +//! +//! use dataset_authoring::package::PackageBuilder; +//! +//! // Build from a directory containing manifest.json, tables/, and optional functions/ +//! let builder = PackageBuilder::from_directory(Path::new("build"))?; +//! builder.write_to(Path::new("dataset.tgz"))?; +//! ``` + +use std::{ + fs::{self, File}, + io::{self, Read}, + path::Path, +}; + +use datasets_common::hash::Hash; +use flate2::{Compression, write::GzEncoder}; + +/// Default file permissions for archive entries (rw-r--r--). +const FILE_MODE: u32 = 0o644; + +/// Default directory permissions for archive entries (rwxr-xr-x). +const DIR_MODE: u32 = 0o755; + +/// Unix epoch timestamp for deterministic archives. +const MTIME: u64 = 0; + +/// Errors that occur during package assembly. +#[derive(Debug, thiserror::Error)] +pub enum PackageError { + /// Failed to read a file to include in the package. + #[error("failed to read file '{}'", path.display())] + ReadFile { + /// Path to the file that could not be read. + path: std::path::PathBuf, + /// Underlying I/O error. + #[source] + source: io::Error, + }, + + /// Failed to write the package archive. + #[error("failed to write package archive")] + WriteArchive(#[source] io::Error), + + /// Failed to create the output file. + #[error("failed to create output file '{}'", path.display())] + CreateOutput { + /// Path to the output file. + path: std::path::PathBuf, + /// Underlying I/O error. + #[source] + source: io::Error, + }, + + /// Failed to finish writing the archive. + #[error("failed to finalize archive")] + FinalizeArchive(#[source] io::Error), + + /// Required manifest.json not found. + #[error("manifest.json not found in package directory")] + MissingManifest, + + /// Failed to read the source directory. + #[error("failed to read directory '{}'", path.display())] + ReadDirectory { + /// Path to the directory. + path: std::path::PathBuf, + /// Underlying I/O error. + #[source] + source: io::Error, + }, +} + +/// An entry to include in the package archive. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackageEntry { + /// Archive path (POSIX style, no leading slash). + pub path: String, + /// File contents. + pub contents: Vec, +} + +impl PackageEntry { + /// Creates a new package entry. + pub fn new(path: impl Into, contents: impl Into>) -> Self { + Self { + path: path.into(), + contents: contents.into(), + } + } + + /// Creates a package entry by reading from a file. + /// + /// # Arguments + /// + /// * `file_path` - Absolute path to the file to read + /// * `archive_path` - Path within the archive (POSIX style) + /// + /// # Errors + /// + /// Returns [`PackageError::ReadFile`] if the file cannot be read. + pub fn from_file( + file_path: &Path, + archive_path: impl Into, + ) -> Result { + let contents = fs::read(file_path).map_err(|err| PackageError::ReadFile { + path: file_path.to_path_buf(), + source: err, + })?; + + Ok(Self { + path: archive_path.into(), + contents, + }) + } +} + +/// Builder for creating deterministic package archives. +/// +/// Collects entries and writes them to a gzipped tar archive with +/// deterministic metadata (sorted entries, fixed timestamps, etc.). +#[derive(Debug, Default)] +pub struct PackageBuilder { + entries: Vec, +} + +impl PackageBuilder { + /// Creates a new empty package builder. + pub fn new() -> Self { + Self { entries: vec![] } + } + + /// Creates a package builder from a directory containing the build output. + /// + /// Expects the directory to contain: + /// - `manifest.json` (required) + /// - `tables/*.sql` and `tables/*.ipc` (table files) + /// - `functions/*.js` (optional function files) + /// + /// # Errors + /// + /// Returns [`PackageError::MissingManifest`] if manifest.json is not found. + /// Returns [`PackageError::ReadDirectory`] if a directory cannot be read. + /// Returns [`PackageError::ReadFile`] if a file cannot be read. + pub fn from_directory(dir: &Path) -> Result { + let mut builder = Self::new(); + + // Add manifest.json (required) + let manifest_path = dir.join("manifest.json"); + if !manifest_path.exists() { + return Err(PackageError::MissingManifest); + } + builder.add_entry(PackageEntry::from_file(&manifest_path, "manifest.json")?); + + // Add tables/ directory contents + let tables_dir = dir.join("tables"); + if tables_dir.exists() { + builder.add_directory_contents(&tables_dir, "tables")?; + } + + // Add functions/ directory contents + let functions_dir = dir.join("functions"); + if functions_dir.exists() { + builder.add_directory_contents(&functions_dir, "functions")?; + } + + Ok(builder) + } + + /// Adds a single entry to the package. + pub fn add_entry(&mut self, entry: PackageEntry) -> &mut Self { + self.entries.push(entry); + self + } + + /// Adds all files from a directory to the package. + /// + /// Files are added with paths relative to the given prefix. + /// + /// # Arguments + /// + /// * `dir` - Directory to read files from + /// * `prefix` - Archive path prefix for the files + fn add_directory_contents( + &mut self, + dir: &Path, + prefix: &str, + ) -> Result<&mut Self, PackageError> { + let entries = fs::read_dir(dir).map_err(|err| PackageError::ReadDirectory { + path: dir.to_path_buf(), + source: err, + })?; + + for entry in entries { + let entry = entry.map_err(|err| PackageError::ReadDirectory { + path: dir.to_path_buf(), + source: err, + })?; + + let path = entry.path(); + if path.is_file() { + let file_name = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("unknown"); + let archive_path = format!("{}/{}", prefix, file_name); + self.add_entry(PackageEntry::from_file(&path, archive_path)?); + } + } + + Ok(self) + } + + /// Returns the entries that will be included in the package. + /// + /// Entries are sorted alphabetically by path for determinism. + pub fn entries(&self) -> Vec<&PackageEntry> { + let mut entries: Vec<_> = self.entries.iter().collect(); + entries.sort_by(|a, b| a.path.cmp(&b.path)); + entries + } + + /// Writes the package archive to a file. + /// + /// # Arguments + /// + /// * `output` - Path to write the archive to + /// + /// # Errors + /// + /// Returns [`PackageError::CreateOutput`] if the output file cannot be created. + /// Returns [`PackageError::WriteArchive`] if writing fails. + /// Returns [`PackageError::FinalizeArchive`] if finalizing the archive fails. + pub fn write_to(&self, output: &Path) -> Result<(), PackageError> { + let file = File::create(output).map_err(|err| PackageError::CreateOutput { + path: output.to_path_buf(), + source: err, + })?; + + self.write(file) + } + + /// Writes the package archive to a writer. + /// + /// This is useful for testing or writing to memory. + /// + /// # Errors + /// + /// Returns [`PackageError::WriteArchive`] if writing fails. + /// Returns [`PackageError::FinalizeArchive`] if finalizing the archive fails. + pub fn write(&self, writer: W) -> Result<(), PackageError> { + let gz = GzEncoder::new(writer, Compression::default()); + let mut tar = tar::Builder::new(gz); + + // Sort entries alphabetically for determinism + let mut sorted_entries: Vec<_> = self.entries.iter().collect(); + sorted_entries.sort_by(|a, b| a.path.cmp(&b.path)); + + // Collect unique directories that need to be created + let mut directories: Vec = vec![]; + for entry in &sorted_entries { + if let Some(parent) = Path::new(&entry.path).parent() { + let parent_str = parent.to_string_lossy(); + if !parent_str.is_empty() && !directories.contains(&parent_str.to_string()) { + directories.push(parent_str.to_string()); + } + } + } + directories.sort(); + + // Add directory entries first + for dir_path in &directories { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_mode(DIR_MODE); + header.set_mtime(MTIME); + header.set_uid(0); + header.set_gid(0); + header.set_cksum(); + + let dir_with_slash = format!("{}/", dir_path); + tar.append_data(&mut header, &dir_with_slash, io::empty()) + .map_err(PackageError::WriteArchive)?; + } + + // Add file entries + for entry in sorted_entries { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(entry.contents.len() as u64); + header.set_mode(FILE_MODE); + header.set_mtime(MTIME); + header.set_uid(0); + header.set_gid(0); + header.set_cksum(); + + tar.append_data(&mut header, &entry.path, entry.contents.as_slice()) + .map_err(PackageError::WriteArchive)?; + } + + // Finish writing + let gz = tar.into_inner().map_err(PackageError::WriteArchive)?; + gz.finish().map_err(PackageError::FinalizeArchive)?; + + Ok(()) + } + + /// Writes the package archive to a byte vector. + /// + /// Useful for computing the hash of the archive or for testing. + /// + /// # Errors + /// + /// Returns [`PackageError::WriteArchive`] if writing fails. + /// Returns [`PackageError::FinalizeArchive`] if finalizing the archive fails. + pub fn to_bytes(&self) -> Result, PackageError> { + let mut buffer = Vec::new(); + self.write(&mut buffer)?; + Ok(buffer) + } + + /// Computes the SHA-256 hash of the package archive. + /// + /// This can be used to verify the package was built correctly + /// or to check for reproducibility. + /// + /// # Errors + /// + /// Returns [`PackageError::WriteArchive`] if writing fails. + /// Returns [`PackageError::FinalizeArchive`] if finalizing the archive fails. + pub fn hash(&self) -> Result { + let bytes = self.to_bytes()?; + Ok(datasets_common::hash::hash(&bytes)) + } +} + +/// Extracts a package archive to a directory. +/// +/// This is useful for testing or inspecting package contents. +/// +/// # Arguments +/// +/// * `archive_path` - Path to the `.tgz` archive +/// * `output_dir` - Directory to extract to (must exist) +/// +/// # Errors +/// +/// Returns [`PackageError::ReadFile`] if the archive cannot be read. +/// Returns [`PackageError::WriteArchive`] if extraction fails. +pub fn extract_to_directory(archive_path: &Path, output_dir: &Path) -> Result<(), PackageError> { + let file = File::open(archive_path).map_err(|err| PackageError::ReadFile { + path: archive_path.to_path_buf(), + source: err, + })?; + + let gz = flate2::read::GzDecoder::new(file); + let mut archive = tar::Archive::new(gz); + + archive + .unpack(output_dir) + .map_err(PackageError::WriteArchive)?; + + Ok(()) +} + +/// Reads the contents of a package archive into memory. +/// +/// Returns a list of entries with their paths and contents. +/// +/// # Arguments +/// +/// * `archive_path` - Path to the `.tgz` archive +/// +/// # Errors +/// +/// Returns [`PackageError::ReadFile`] if the archive cannot be read. +/// Returns [`PackageError::WriteArchive`] if parsing fails. +pub fn read_archive(archive_path: &Path) -> Result, PackageError> { + let file = File::open(archive_path).map_err(|err| PackageError::ReadFile { + path: archive_path.to_path_buf(), + source: err, + })?; + + read_archive_from_reader(file) +} + +/// Reads the contents of a package archive from a reader. +/// +/// Returns a list of entries with their paths and contents. +/// +/// # Errors +/// +/// Returns [`PackageError::WriteArchive`] if parsing fails. +pub fn read_archive_from_reader(reader: R) -> Result, PackageError> { + let gz = flate2::read::GzDecoder::new(reader); + let mut archive = tar::Archive::new(gz); + + let mut entries = Vec::new(); + + for entry in archive.entries().map_err(PackageError::WriteArchive)? { + let mut entry = entry.map_err(PackageError::WriteArchive)?; + + // Skip directories + if entry.header().entry_type() == tar::EntryType::Directory { + continue; + } + + let path = entry + .path() + .map_err(PackageError::WriteArchive)? + .to_string_lossy() + .into_owned(); + + let mut contents = Vec::new(); + entry + .read_to_end(&mut contents) + .map_err(PackageError::WriteArchive)?; + + entries.push(PackageEntry { path, contents }); + } + + // Sort for consistent ordering + entries.sort_by(|a, b| a.path.cmp(&b.path)); + + Ok(entries) +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use super::*; + + // ============================================================ + // PackageEntry tests + // ============================================================ + + #[test] + fn package_entry_new_creates_entry() { + //* Given + let path = "manifest.json"; + let contents = b"{}".to_vec(); + + //* When + let entry = PackageEntry::new(path, contents.clone()); + + //* Then + assert_eq!(entry.path, path); + assert_eq!(entry.contents, contents); + } + + #[test] + fn package_entry_from_file_reads_content() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let file_path = dir.path().join("test.txt"); + fs::write(&file_path, b"hello world").expect("should write file"); + + //* When + let result = PackageEntry::from_file(&file_path, "test.txt"); + + //* Then + let entry = result.expect("should create entry"); + assert_eq!(entry.path, "test.txt"); + assert_eq!(entry.contents, b"hello world"); + } + + #[test] + fn package_entry_from_nonexistent_file_fails() { + //* Given + let path = Path::new("/nonexistent/file.txt"); + + //* When + let result = PackageEntry::from_file(path, "file.txt"); + + //* Then + assert!(result.is_err()); + let err = result.expect_err("should fail"); + assert!(matches!(err, PackageError::ReadFile { .. })); + } + + // ============================================================ + // PackageBuilder tests + // ============================================================ + + #[test] + fn package_builder_new_creates_empty() { + //* Given / When + let builder = PackageBuilder::new(); + + //* Then + assert!(builder.entries.is_empty()); + } + + #[test] + fn package_builder_add_entry_adds_entry() { + //* Given + let mut builder = PackageBuilder::new(); + let entry = PackageEntry::new("manifest.json", b"{}".to_vec()); + + //* When + builder.add_entry(entry); + + //* Then + assert_eq!(builder.entries.len(), 1); + } + + #[test] + fn package_builder_entries_returns_sorted() { + //* Given + let mut builder = PackageBuilder::new(); + builder.add_entry(PackageEntry::new("tables/z.sql", b"z")); + builder.add_entry(PackageEntry::new("manifest.json", b"{}")); + builder.add_entry(PackageEntry::new("tables/a.sql", b"a")); + + //* When + let entries = builder.entries(); + + //* Then + assert_eq!(entries.len(), 3); + assert_eq!(entries[0].path, "manifest.json"); + assert_eq!(entries[1].path, "tables/a.sql"); + assert_eq!(entries[2].path, "tables/z.sql"); + } + + #[test] + fn package_builder_to_bytes_produces_valid_archive() { + //* Given + let mut builder = PackageBuilder::new(); + builder.add_entry(PackageEntry::new("manifest.json", b"{}".to_vec())); + + //* When + let result = builder.to_bytes(); + + //* Then + let bytes = result.expect("should produce bytes"); + assert!(!bytes.is_empty()); + // Should start with gzip magic number + assert_eq!(&bytes[0..2], &[0x1f, 0x8b]); + } + + #[test] + fn package_builder_produces_deterministic_output() { + //* Given + let mut builder1 = PackageBuilder::new(); + builder1.add_entry(PackageEntry::new("manifest.json", b"{}".to_vec())); + builder1.add_entry(PackageEntry::new("tables/test.sql", b"SELECT 1".to_vec())); + + let mut builder2 = PackageBuilder::new(); + // Add in different order + builder2.add_entry(PackageEntry::new("tables/test.sql", b"SELECT 1".to_vec())); + builder2.add_entry(PackageEntry::new("manifest.json", b"{}".to_vec())); + + //* When + let bytes1 = builder1.to_bytes().expect("should produce bytes"); + let bytes2 = builder2.to_bytes().expect("should produce bytes"); + + //* Then + assert_eq!( + bytes1, bytes2, + "archives should be identical regardless of entry order" + ); + } + + #[test] + fn package_builder_hash_is_deterministic() { + //* Given + let mut builder = PackageBuilder::new(); + builder.add_entry(PackageEntry::new("manifest.json", b"{}".to_vec())); + + //* When + let hash1 = builder.hash().expect("should compute hash"); + let hash2 = builder.hash().expect("should compute hash"); + + //* Then + assert_eq!(hash1, hash2, "hash should be deterministic"); + } + + #[test] + fn package_builder_from_directory_requires_manifest() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + // Don't create manifest.json + + //* When + let result = PackageBuilder::from_directory(dir.path()); + + //* Then + assert!(result.is_err()); + let err = result.expect_err("should fail"); + assert!(matches!(err, PackageError::MissingManifest)); + } + + #[test] + fn package_builder_from_directory_includes_all_files() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + + // Create manifest.json + fs::write(dir.path().join("manifest.json"), b"{}").expect("should write"); + + // Create tables/ directory with files + fs::create_dir(dir.path().join("tables")).expect("should create tables dir"); + fs::write(dir.path().join("tables/users.sql"), b"SELECT * FROM users") + .expect("should write"); + fs::write(dir.path().join("tables/users.ipc"), b"ipc schema data").expect("should write"); + + // Create functions/ directory with file + fs::create_dir(dir.path().join("functions")).expect("should create functions dir"); + fs::write( + dir.path().join("functions/decode.js"), + b"export function decode() {}", + ) + .expect("should write"); + + //* When + let result = PackageBuilder::from_directory(dir.path()); + + //* Then + let builder = result.expect("should build from directory"); + let entries = builder.entries(); + + // Should have 4 files + assert_eq!(entries.len(), 4); + + // Verify paths (sorted) + let paths: Vec<_> = entries.iter().map(|e| e.path.as_str()).collect(); + assert!(paths.contains(&"manifest.json")); + assert!(paths.contains(&"tables/users.sql")); + assert!(paths.contains(&"tables/users.ipc")); + assert!(paths.contains(&"functions/decode.js")); + } + + #[test] + fn package_builder_write_to_creates_file() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let output_path = dir.path().join("dataset.tgz"); + + let mut builder = PackageBuilder::new(); + builder.add_entry(PackageEntry::new("manifest.json", b"{}".to_vec())); + + //* When + let result = builder.write_to(&output_path); + + //* Then + assert!(result.is_ok()); + assert!(output_path.exists()); + } + + // ============================================================ + // Archive reading tests + // ============================================================ + + #[test] + fn roundtrip_write_and_read_archive() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let archive_path = dir.path().join("test.tgz"); + + let mut builder = PackageBuilder::new(); + builder.add_entry(PackageEntry::new("manifest.json", b"{\"name\":\"test\"}")); + builder.add_entry(PackageEntry::new( + "tables/users.sql", + b"SELECT * FROM users", + )); + + builder.write_to(&archive_path).expect("should write"); + + //* When + let result = read_archive(&archive_path); + + //* Then + let entries = result.expect("should read archive"); + assert_eq!(entries.len(), 2); + + assert_eq!(entries[0].path, "manifest.json"); + assert_eq!(entries[0].contents, b"{\"name\":\"test\"}"); + + assert_eq!(entries[1].path, "tables/users.sql"); + assert_eq!(entries[1].contents, b"SELECT * FROM users"); + } + + #[test] + fn extract_to_directory_creates_files() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let archive_path = dir.path().join("test.tgz"); + let output_dir = dir.path().join("extracted"); + fs::create_dir(&output_dir).expect("should create output dir"); + + let mut builder = PackageBuilder::new(); + builder.add_entry(PackageEntry::new("manifest.json", b"{}")); + builder.add_entry(PackageEntry::new("tables/users.sql", b"SELECT 1")); + builder.write_to(&archive_path).expect("should write"); + + //* When + let result = extract_to_directory(&archive_path, &output_dir); + + //* Then + assert!(result.is_ok()); + assert!(output_dir.join("manifest.json").exists()); + assert!(output_dir.join("tables/users.sql").exists()); + } + + // ============================================================ + // Determinism verification tests + // ============================================================ + + #[test] + fn multiple_builds_produce_identical_archives() { + //* Given + let entries = vec![ + PackageEntry::new("manifest.json", b"{}".to_vec()), + PackageEntry::new("tables/a.sql", b"SELECT 1".to_vec()), + PackageEntry::new("tables/b.sql", b"SELECT 2".to_vec()), + PackageEntry::new("functions/f.js", b"export function f() {}".to_vec()), + ]; + + //* When + let mut hashes = Vec::new(); + for _ in 0..3 { + let mut builder = PackageBuilder::new(); + for entry in &entries { + builder.add_entry(entry.clone()); + } + hashes.push(builder.hash().expect("should compute hash")); + } + + //* Then + assert_eq!(hashes[0], hashes[1]); + assert_eq!(hashes[1], hashes[2]); + } +} diff --git a/crates/core/dataset-authoring/src/query.rs b/crates/core/dataset-authoring/src/query.rs new file mode 100644 index 000000000..c94e1f5cd --- /dev/null +++ b/crates/core/dataset-authoring/src/query.rs @@ -0,0 +1,496 @@ +//! SELECT statement validation for dataset authoring. +//! +//! This module validates that SQL model files contain exactly one SELECT statement, +//! rejecting CTAS, DDL, and DML statements. +//! +//! # SELECT Requirements +//! +//! For derived datasets, SQL model files must: +//! - Contain exactly one `SELECT ...` statement +//! - Not use CTAS (`CREATE TABLE ... AS SELECT`) +//! - Not use DDL (`CREATE TABLE`, `CREATE INDEX`, etc.) +//! - Not use DML (`INSERT`, `UPDATE`, `DELETE`) +//! +//! # Example +//! +//! ```sql +//! -- Valid SELECT +//! SELECT block_num, tx_hash, from_address, to_address, value +//! FROM source.transactions +//! WHERE value > 0 +//! ``` +//! +//! # Note +//! +//! Table names are derived from file names via model discovery, not from SQL content. + +use datafusion::sql::{parser::Statement, sqlparser::ast}; +use datasets_common::table_name::TableName; + +/// Validated SELECT information extracted from a SQL statement. +#[derive(Debug, Clone)] +pub struct SelectInfo { + /// The table name (from discovery, not SQL). + pub table_name: TableName, + /// The SELECT query. + pub query: Box, +} + +/// Errors that occur when validating SELECT statements. +#[derive(Debug, thiserror::Error)] +pub enum SelectValidationError { + /// The SQL statement is not a SELECT query. + /// + /// This error occurs when the parsed SQL is a DDL statement (CREATE TABLE, etc.), + /// DML statement (INSERT, UPDATE, DELETE), or other non-query statement. + /// Dataset models must contain only SELECT queries. + #[error("expected SELECT statement, found {found}")] + NotSelect { + /// Description of what statement type was found. + found: String, + }, +} + +/// Validates that a statement is a SELECT query and extracts the query. +/// +/// This function validates that the parsed statement is a pure SELECT query, +/// rejecting CTAS, DDL, and DML statements. The table name is provided externally +/// from model discovery (based on filename), not extracted from the SQL. +/// +/// # Arguments +/// +/// * `stmt` - A parsed DataFusion statement to validate. +/// * `table_name` - The table name from model discovery (file stem). +/// +/// # Returns +/// +/// * `Ok(SelectInfo)` - The statement is a valid SELECT query. +/// * `Err(SelectValidationError::NotSelect)` - The statement is not a SELECT. +/// +/// # Examples +/// +/// ```ignore +/// use common::sql::parse; +/// use dataset_authoring::query::validate_select; +/// use datasets_common::table_name::TableName; +/// +/// let sql = "SELECT * FROM source.txs".parse().unwrap(); +/// let stmt = parse(&sql).unwrap(); +/// let table_name: TableName = "transfers".parse().unwrap(); +/// let select_info = validate_select(&stmt, table_name).unwrap(); +/// assert_eq!(select_info.table_name.as_str(), "transfers"); +/// ``` +pub fn validate_select( + stmt: &Statement, + table_name: TableName, +) -> Result { + match stmt { + Statement::Statement(inner) => match inner.as_ref() { + ast::Statement::Query(query) => Ok(SelectInfo { + table_name, + query: query.clone(), + }), + ast::Statement::CreateTable(_) => Err(SelectValidationError::NotSelect { + found: "CREATE TABLE".to_string(), + }), + ast::Statement::Insert(_) => Err(SelectValidationError::NotSelect { + found: "INSERT".to_string(), + }), + ast::Statement::Update { .. } => Err(SelectValidationError::NotSelect { + found: "UPDATE".to_string(), + }), + ast::Statement::Delete(_) => Err(SelectValidationError::NotSelect { + found: "DELETE".to_string(), + }), + ast::Statement::CreateView { .. } => Err(SelectValidationError::NotSelect { + found: "CREATE VIEW".to_string(), + }), + ast::Statement::CreateIndex(_) => Err(SelectValidationError::NotSelect { + found: "CREATE INDEX".to_string(), + }), + ast::Statement::AlterTable { .. } => Err(SelectValidationError::NotSelect { + found: "ALTER TABLE".to_string(), + }), + ast::Statement::Drop { .. } => Err(SelectValidationError::NotSelect { + found: "DROP".to_string(), + }), + ast::Statement::Truncate { .. } => Err(SelectValidationError::NotSelect { + found: "TRUNCATE".to_string(), + }), + other => Err(SelectValidationError::NotSelect { + found: statement_kind_description(other), + }), + }, + Statement::CreateExternalTable(_) => Err(SelectValidationError::NotSelect { + found: "CREATE EXTERNAL TABLE".to_string(), + }), + Statement::CopyTo(_) => Err(SelectValidationError::NotSelect { + found: "COPY TO".to_string(), + }), + Statement::Explain(_) => Err(SelectValidationError::NotSelect { + found: "EXPLAIN".to_string(), + }), + Statement::Reset(_) => Err(SelectValidationError::NotSelect { + found: "RESET".to_string(), + }), + } +} + +/// Returns a human-readable description of a sqlparser Statement kind. +fn statement_kind_description(stmt: &ast::Statement) -> String { + // Extract the variant name from Debug output + format!("{:?}", std::mem::discriminant(stmt)) + .split('(') + .next() + .unwrap_or("unknown statement") + .to_string() +} + +#[cfg(test)] +mod tests { + use common::sql::parse; + use datasets_derived::sql_str::SqlStr; + + use super::*; + + /// Helper to parse SQL into a Statement + fn parse_sql(sql: &str) -> Statement { + let sql_str: SqlStr = sql.parse().expect("SQL string should be valid"); + parse(&sql_str).expect("SQL should parse successfully") + } + + /// Helper to create a table name for tests + fn test_table_name(name: &str) -> TableName { + name.parse().expect("table name should be valid") + } + + mod valid_select_tests { + use super::*; + + #[test] + fn valid_simple_select() { + //* Given + let stmt = parse_sql("SELECT id, name FROM t"); + let table_name = test_table_name("my_table"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let info = result.expect("should validate successfully"); + assert_eq!( + info.table_name.as_str(), + "my_table", + "table name should match provided name" + ); + } + + #[test] + fn valid_select_star() { + //* Given + let stmt = parse_sql("SELECT * FROM source"); + let table_name = test_table_name("transfers"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let info = result.expect("should validate successfully"); + assert_eq!(info.table_name.as_str(), "transfers"); + } + + #[test] + fn valid_select_with_where() { + //* Given + let stmt = parse_sql("SELECT * FROM t WHERE x > 0"); + let table_name = test_table_name("filtered"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + result.expect("should validate SELECT with WHERE"); + } + + #[test] + fn valid_select_with_joins() { + //* Given + let stmt = parse_sql( + "SELECT t.block_num, t.tx_hash, b.timestamp \ + FROM source.transactions t \ + JOIN source.blocks b ON t.block_num = b.number", + ); + let table_name = test_table_name("joined"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let info = result.expect("should validate SELECT with JOIN"); + assert_eq!(info.table_name.as_str(), "joined"); + } + + #[test] + fn valid_select_with_cte() { + //* Given + let stmt = parse_sql( + "WITH filtered AS (SELECT * FROM source WHERE value > 0) \ + SELECT * FROM filtered", + ); + let table_name = test_table_name("result"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let info = result.expect("should validate SELECT with CTE"); + assert_eq!(info.table_name.as_str(), "result"); + } + + #[test] + fn valid_select_with_subquery() { + //* Given + let stmt = parse_sql( + "SELECT * FROM (SELECT id, value FROM source WHERE value > 100) AS filtered", + ); + let table_name = test_table_name("subquery_result"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + result.expect("should validate SELECT with subquery"); + } + + #[test] + fn valid_select_with_union() { + //* Given + let stmt = + parse_sql("SELECT id, name FROM table_a UNION ALL SELECT id, name FROM table_b"); + let table_name = test_table_name("combined"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + result.expect("should validate SELECT with UNION"); + } + } + + mod reject_ctas_tests { + use super::*; + + #[test] + fn rejects_simple_ctas() { + //* Given + let stmt = parse_sql("CREATE TABLE transfers AS SELECT * FROM source"); + let table_name = test_table_name("transfers"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let err = result.expect_err("should reject CTAS"); + assert!( + matches!(&err, SelectValidationError::NotSelect { found } if found == "CREATE TABLE"), + "expected NotSelect error for CREATE TABLE, got: {:?}", + err + ); + } + + #[test] + fn rejects_ctas_with_complex_select() { + //* Given + let stmt = parse_sql( + "CREATE TABLE filtered AS \ + SELECT block_num, tx_hash FROM source.transactions WHERE value > 0", + ); + let table_name = test_table_name("filtered"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let err = result.expect_err("should reject CTAS with complex SELECT"); + assert!( + matches!(&err, SelectValidationError::NotSelect { found } if found == "CREATE TABLE"), + "expected NotSelect error, got: {:?}", + err + ); + } + } + + mod reject_ddl_tests { + use super::*; + + #[test] + fn rejects_create_table() { + //* Given + let stmt = parse_sql("CREATE TABLE test (id INT, name VARCHAR)"); + let table_name = test_table_name("test"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let err = result.expect_err("should reject CREATE TABLE"); + assert!( + matches!(&err, SelectValidationError::NotSelect { found } if found == "CREATE TABLE"), + "expected NotSelect error, got: {:?}", + err + ); + } + + #[test] + fn rejects_create_view() { + //* Given + let stmt = parse_sql("CREATE VIEW test AS SELECT * FROM source"); + let table_name = test_table_name("test"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let err = result.expect_err("should reject CREATE VIEW"); + assert!( + matches!(&err, SelectValidationError::NotSelect { found } if found == "CREATE VIEW"), + "expected NotSelect error, got: {:?}", + err + ); + } + + #[test] + fn rejects_create_index() { + //* Given + let stmt = parse_sql("CREATE INDEX idx ON test (id)"); + let table_name = test_table_name("test"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let err = result.expect_err("should reject CREATE INDEX"); + assert!( + matches!(&err, SelectValidationError::NotSelect { found } if found == "CREATE INDEX"), + "expected NotSelect error, got: {:?}", + err + ); + } + + #[test] + fn rejects_drop() { + //* Given + let stmt = parse_sql("DROP TABLE test"); + let table_name = test_table_name("test"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let err = result.expect_err("should reject DROP"); + assert!( + matches!(&err, SelectValidationError::NotSelect { found } if found == "DROP"), + "expected NotSelect error, got: {:?}", + err + ); + } + } + + mod reject_dml_tests { + use super::*; + + #[test] + fn rejects_insert() { + //* Given + let stmt = parse_sql("INSERT INTO test VALUES (1, 'a')"); + let table_name = test_table_name("test"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let err = result.expect_err("should reject INSERT"); + assert!( + matches!(&err, SelectValidationError::NotSelect { found } if found == "INSERT"), + "expected NotSelect error, got: {:?}", + err + ); + } + + #[test] + fn rejects_update() { + //* Given + let stmt = parse_sql("UPDATE test SET name = 'b' WHERE id = 1"); + let table_name = test_table_name("test"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let err = result.expect_err("should reject UPDATE"); + assert!( + matches!(&err, SelectValidationError::NotSelect { found } if found == "UPDATE"), + "expected NotSelect error, got: {:?}", + err + ); + } + + #[test] + fn rejects_delete() { + //* Given + let stmt = parse_sql("DELETE FROM test WHERE id = 1"); + let table_name = test_table_name("test"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let err = result.expect_err("should reject DELETE"); + assert!( + matches!(&err, SelectValidationError::NotSelect { found } if found == "DELETE"), + "expected NotSelect error, got: {:?}", + err + ); + } + + #[test] + fn rejects_truncate() { + //* Given + let stmt = parse_sql("TRUNCATE TABLE test"); + let table_name = test_table_name("test"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let err = result.expect_err("should reject TRUNCATE"); + assert!( + matches!(&err, SelectValidationError::NotSelect { found } if found == "TRUNCATE"), + "expected NotSelect error, got: {:?}", + err + ); + } + } + + mod other_statement_tests { + use super::*; + + #[test] + fn rejects_explain() { + //* Given + let stmt = parse_sql("EXPLAIN SELECT * FROM test"); + let table_name = test_table_name("test"); + + //* When + let result = validate_select(&stmt, table_name); + + //* Then + let err = result.expect_err("should reject EXPLAIN"); + assert!( + matches!(&err, SelectValidationError::NotSelect { found } if found == "EXPLAIN"), + "expected NotSelect error, got: {:?}", + err + ); + } + } +} diff --git a/crates/core/dataset-authoring/src/resolver.rs b/crates/core/dataset-authoring/src/resolver.rs new file mode 100644 index 000000000..ea1a90c19 --- /dev/null +++ b/crates/core/dataset-authoring/src/resolver.rs @@ -0,0 +1,1231 @@ +//! Dependency resolution for dataset authoring. +//! +//! Resolves dataset dependencies from a registry (admin API) and caches +//! them locally. Handles both version-based and hash-based references, +//! with cycle detection in the dependency graph. +//! +//! # Resolution Process +//! +//! 1. Parse dependency references from `amp.yaml` +//! 2. Check local cache for each dependency +//! 3. Fetch missing dependencies from admin API +//! 4. Compute and verify content hashes +//! 5. Build transitive dependency graph +//! 6. Detect cycles using DFS +//! +//! # Example +//! +//! ```ignore +//! use dataset_authoring::resolver::{Resolver, ResolverConfig}; +//! use admin_client::Client; +//! +//! let client = Client::new(base_url); +//! let resolver = Resolver::new(client, ResolverConfig::default()); +//! +//! let graph = resolver.resolve_all(&dependencies).await?; +//! ``` + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use async_trait::async_trait; +use datasets_common::{ + hash::Hash, + name::{Name, NameError}, + namespace::{Namespace, NamespaceError}, +}; +use datasets_derived::deps::{DepAlias, DepReference, HashOrVersion}; + +use crate::{ + adapter::{AdapterError, LegacyAdapter}, + cache::{Cache, CacheError}, + canonical::{CanonicalizeError, hash_canonical}, + dependency_manifest::DependencyManifest, + lockfile::Lockfile, +}; + +/// Errors that occur during dependency resolution. +#[derive(Debug, thiserror::Error)] +pub enum ResolveError { + /// Dependency not found in registry. + /// + /// This occurs when a referenced dataset does not exist in the registry + /// at the specified version or hash. + #[error("dependency not found: {reference}")] + NotFound { + /// The reference that could not be found + reference: String, + }, + + /// Network or API error during resolution. + /// + /// This occurs when communication with the registry fails due to + /// network issues, authentication problems, or server errors. + #[error("failed to fetch dependency '{reference}'")] + FetchError { + /// The reference being fetched + reference: String, + /// The underlying error message + message: String, + }, + + /// Circular dependency detected in the graph. + /// + /// This occurs when the dependency graph contains a cycle, which would + /// cause infinite resolution or runtime loops. + #[error("circular dependency detected: {path}")] + CycleDetected { + /// A string representation of the cycle path + path: String, + }, + + /// Hash mismatch between declared and computed hash. + /// + /// This occurs when a dependency references a specific hash but the + /// fetched manifest's computed hash does not match. + #[error("hash mismatch for '{reference}': expected {expected}, got {actual}")] + HashMismatch { + /// The reference being resolved + reference: String, + /// The expected hash from the reference + expected: Hash, + /// The actual computed hash of the manifest + actual: Hash, + }, + + /// Failed to adapt legacy manifest from registry. + /// + /// This occurs when the registry returns a derived manifest that cannot be + /// converted to the canonical package format. + #[error("failed to adapt legacy manifest for '{reference}'")] + AdaptError { + /// The reference being resolved + reference: String, + /// The underlying adapter error + #[source] + source: AdapterError, + }, + + /// Failed to parse raw manifest from registry. + /// + /// This occurs when the registry returns a raw manifest that cannot be + /// parsed into a DependencyManifest. + #[error("failed to parse raw manifest for '{reference}'")] + ParseRawManifest { + /// The reference being resolved + reference: String, + /// The underlying parse error + #[source] + source: serde_json::Error, + }, + + /// Failed to canonicalize manifest for hashing. + /// + /// This occurs when the manifest cannot be converted to canonical JSON + /// for hash computation. + #[error("failed to canonicalize manifest for '{reference}'")] + CanonicalizeError { + /// The reference being resolved + reference: String, + /// The underlying canonicalization error + #[source] + source: CanonicalizeError, + }, + + /// Cache operation failed. + /// + /// This occurs when reading from or writing to the local cache fails. + #[error("cache error for hash '{hash}'")] + CacheError { + /// The hash involved in the cache operation + hash: Hash, + /// The underlying cache error + #[source] + source: CacheError, + }, + + /// Dependency is missing from cache in offline mode. + /// + /// This occurs when offline resolution is requested and the dependency + /// is not available in the local cache. + #[error("dependency '{0}' not in cache (offline mode)")] + OfflineCacheMiss(String), + + /// Version references cannot be resolved in offline mode without a lockfile. + /// + /// This occurs when a dependency reference uses a version and offline mode + /// is enabled, requiring an amp.lock mapping or hash pinning. + #[error( + "offline mode requires amp.lock for version reference '{0}'; add amp.lock, pin by hash, or run without --offline" + )] + OfflineVersionReference(String), + + /// Lockfile node has an invalid namespace value. + /// + /// This occurs when a lockfile node's namespace cannot be parsed into a + /// valid [`Namespace`], which indicates a corrupted or invalid lockfile. + #[error("invalid namespace '{namespace}' in lockfile for hash '{hash}'")] + InvalidLockfileNamespace { + /// The dependency hash with invalid namespace metadata. + hash: Hash, + /// The namespace string from the lockfile. + namespace: String, + /// The underlying namespace parsing error. + #[source] + source: NamespaceError, + }, + + /// Lockfile node has an invalid name value. + /// + /// This occurs when a lockfile node's name cannot be parsed into a + /// valid [`Name`], which indicates a corrupted or invalid lockfile. + #[error("invalid name '{name}' in lockfile for hash '{hash}'")] + InvalidLockfileName { + /// The dependency hash with invalid name metadata. + hash: Hash, + /// The name string from the lockfile. + name: String, + /// The underlying name parsing error. + #[source] + source: NameError, + }, +} + +/// A resolved dependency with its manifest and computed hash. +#[derive(Debug, Clone)] +pub struct ResolvedDependency { + /// The content hash of the manifest (identity). + pub hash: Hash, + /// The resolved manifest. + pub manifest: DependencyManifest, + /// The namespace extracted from the dependency reference. + pub namespace: Namespace, + /// The name extracted from the dependency reference. + pub name: Name, +} + +/// Node in the dependency graph. +#[derive(Debug, Clone)] +pub struct DependencyNode { + /// The resolved dependency information. + pub resolved: ResolvedDependency, + /// Hashes of direct dependencies (from manifest.dependencies). + pub deps: Vec, +} + +/// Complete resolved dependency graph. +/// +/// Contains all transitive dependencies keyed by their content hash, +/// plus the mapping from top-level aliases to their resolved hashes. +#[derive(Debug, Clone)] +pub struct DependencyGraph { + /// Direct dependencies mapped from alias to hash. + pub direct: BTreeMap, + /// All nodes in the graph (transitive closure) keyed by hash. + pub nodes: BTreeMap, +} + +impl DependencyGraph { + /// Creates an empty dependency graph. + pub fn new() -> Self { + Self { + direct: BTreeMap::new(), + nodes: BTreeMap::new(), + } + } + + /// Returns the number of nodes in the graph (transitive dependencies). + pub fn len(&self) -> usize { + self.nodes.len() + } + + /// Returns true if the graph has no dependencies. + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + /// Gets a dependency by its hash. + pub fn get(&self, hash: &Hash) -> Option<&DependencyNode> { + self.nodes.get(hash) + } + + /// Gets a direct dependency by its alias. + pub fn get_by_alias(&self, alias: &DepAlias) -> Option<&DependencyNode> { + self.direct.get(alias).and_then(|h| self.nodes.get(h)) + } + + /// Returns an iterator over all dependency hashes in topological order. + /// + /// Dependencies are ordered such that each dependency appears after + /// all of its own dependencies. + pub fn topological_order(&self) -> Result, ResolveError> { + let mut ordered = Vec::new(); + let mut visited = BTreeSet::new(); + let mut visiting = BTreeSet::new(); + + // Build adjacency list + let deps_map: BTreeMap> = self + .nodes + .iter() + .map(|(hash, node)| (hash.clone(), node.deps.clone())) + .collect(); + + for hash in self.nodes.keys() { + self.dfs_visit(hash, &deps_map, &mut ordered, &mut visited, &mut visiting)?; + } + + Ok(ordered) + } + + fn dfs_visit( + &self, + node: &Hash, + deps: &BTreeMap>, + ordered: &mut Vec, + visited: &mut BTreeSet, + visiting: &mut BTreeSet, + ) -> Result<(), ResolveError> { + if visiting.contains(node) { + return Err(ResolveError::CycleDetected { + path: node.to_string(), + }); + } + if visited.contains(node) { + return Ok(()); + } + + visiting.insert(node.clone()); + + if let Some(node_deps) = deps.get(node) { + for dep in node_deps { + self.dfs_visit(dep, deps, ordered, visited, visiting)?; + } + } + + visiting.remove(node); + visited.insert(node.clone()); + ordered.push(node.clone()); + + Ok(()) + } +} + +impl Default for DependencyGraph { + fn default() -> Self { + Self::new() + } +} + +/// Trait for fetching manifests from a registry. +/// +/// This abstraction allows for different registry implementations +/// (admin API, mock for testing, etc.). +#[async_trait] +pub trait ManifestFetcher: Send + Sync { + /// Fetches a manifest by version reference. + /// + /// The reference format is `namespace/name@version`. + async fn fetch_by_version( + &self, + reference: &DepReference, + ) -> Result; + + /// Fetches a manifest by content hash. + async fn fetch_by_hash(&self, hash: &Hash) -> Result, ResolveError>; +} + +/// Dependency resolver for dataset authoring. +/// +/// Resolves dependencies from a registry and caches them locally. +pub struct Resolver { + /// The manifest fetcher (registry client). + fetcher: F, + /// Local cache for resolved manifests. + cache: Cache, + /// Whether network fetches are disabled. + offline: bool, +} + +impl Resolver { + /// Creates a new resolver with the given fetcher and cache. + pub fn new(fetcher: F, cache: Cache) -> Self { + Self { + fetcher, + cache, + offline: false, + } + } + + /// Creates a new resolver with the given fetcher and default cache location. + /// + /// # Errors + /// + /// Returns [`CacheError::NoHomeDir`] if the home directory cannot be determined. + pub fn with_default_cache(fetcher: F) -> Result { + let cache = Cache::new()?; + Ok(Self { + fetcher, + cache, + offline: false, + }) + } + + /// Sets whether the resolver should operate in offline mode. + pub fn with_offline(mut self, offline: bool) -> Self { + self.offline = offline; + self + } + + /// Resolves a single dependency reference to its manifest and hash. + /// + /// This method handles both version-based and hash-based references: + /// - Version references are resolved via the registry + /// - Hash references are checked in cache first, then fetched if needed + /// + /// # Errors + /// + /// Returns an error if the dependency cannot be found, fetched, or parsed. + pub async fn resolve_one( + &self, + reference: &DepReference, + ) -> Result { + match reference.revision() { + HashOrVersion::Hash(expected_hash) => { + let namespace = reference.fqn().namespace().clone(); + let name = reference.fqn().name().clone(); + let reference = reference.to_string(); + self.resolve_hash(expected_hash, namespace, name, reference) + .await + } + HashOrVersion::Version(_) => { + if self.offline { + return Err(ResolveError::OfflineVersionReference(reference.to_string())); + } + // Fetch by version from registry + let json = self.fetcher.fetch_by_version(reference).await?; + + // Compute hash of fetched manifest + let hash = hash_canonical(&json).map_err(|e| ResolveError::CanonicalizeError { + reference: reference.to_string(), + source: e, + })?; + + // Check cache using computed hash + if let Some(manifest) = + self.cache + .get(&hash) + .map_err(|e| ResolveError::CacheError { + hash: hash.clone(), + source: e, + })? + { + return Ok(ResolvedDependency { + hash, + manifest, + namespace: reference.fqn().namespace().clone(), + name: reference.fqn().name().clone(), + }); + } + + // Adapt legacy format and cache + let manifest = self.adapt_and_cache(&json, &hash, &reference.to_string())?; + + Ok(ResolvedDependency { + hash, + manifest, + namespace: reference.fqn().namespace().clone(), + name: reference.fqn().name().clone(), + }) + } + } + } + + /// Resolves dependencies using hashes from a lockfile. + /// + /// This uses the lockfile as the source of truth for dependency hashes and + /// fetches manifests by hash only (no version resolution). + /// + /// # Errors + /// + /// Returns an error if any dependency cannot be fetched or parsed. + pub async fn resolve_all_locked( + &self, + lockfile: &Lockfile, + ) -> Result { + let mut graph = DependencyGraph::new(); + graph.direct = lockfile.dependencies.clone(); + + for (hash, node) in &lockfile.nodes { + let namespace: Namespace = + node.namespace + .parse() + .map_err(|err| ResolveError::InvalidLockfileNamespace { + hash: hash.clone(), + namespace: node.namespace.clone(), + source: err, + })?; + let name: Name = + node.name + .parse() + .map_err(|err| ResolveError::InvalidLockfileName { + hash: hash.clone(), + name: node.name.clone(), + source: err, + })?; + + let reference = format!("{namespace}/{name}@{hash}"); + let resolved = self.resolve_hash(hash, namespace, name, reference).await?; + + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved, + deps: node.deps.clone(), + }, + ); + } + + graph.topological_order()?; + + Ok(graph) + } + + async fn resolve_hash( + &self, + expected_hash: &Hash, + namespace: Namespace, + name: Name, + reference: String, + ) -> Result { + if let Some(manifest) = + self.cache + .get(expected_hash) + .map_err(|err| ResolveError::CacheError { + hash: expected_hash.clone(), + source: err, + })? + { + return Ok(ResolvedDependency { + hash: expected_hash.clone(), + manifest, + namespace, + name, + }); + } + + if self.offline { + return Err(ResolveError::OfflineCacheMiss(reference)); + } + + let json = self + .fetcher + .fetch_by_hash(expected_hash) + .await? + .ok_or_else(|| ResolveError::NotFound { + reference: reference.clone(), + })?; + + let actual_hash = hash_canonical(&json).map_err(|err| ResolveError::CanonicalizeError { + reference: reference.clone(), + source: err, + })?; + + if &actual_hash != expected_hash { + return Err(ResolveError::HashMismatch { + reference, + expected: expected_hash.clone(), + actual: actual_hash, + }); + } + + // Adapt legacy format and cache + let manifest = self.adapt_and_cache(&json, expected_hash, &reference)?; + + Ok(ResolvedDependency { + hash: expected_hash.clone(), + manifest, + namespace, + name, + }) + } + + /// Adapts a legacy manifest JSON and caches the result. + /// + /// For derived datasets (kind="manifest"), uses [`LegacyAdapter`] to convert + /// the legacy inline format to canonical package format, writing IPC schema + /// files to the cache directory. + /// + /// For raw datasets (evm-rpc, firehose, etc.), parses directly into + /// [`DependencyManifest`] since they don't have inline SQL to extract. + fn adapt_and_cache( + &self, + json: &serde_json::Value, + hash: &Hash, + reference: &str, + ) -> Result { + // Check the manifest kind to determine how to process it + let kind = json + .get("kind") + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + + let manifest = if kind == "manifest" { + // Derived dataset: use adapter to extract inline content to files + let target_dir = self.cache.dir_path(hash); + + // Create cache directory if needed + if !target_dir.exists() { + std::fs::create_dir_all(&target_dir).map_err(|err| ResolveError::CacheError { + hash: hash.clone(), + source: CacheError::CreateDir { + path: target_dir.clone(), + source: err, + }, + })?; + } + + // Use the adapter to convert legacy format to canonical package + let adapter = LegacyAdapter::new(&target_dir); + let adapted = + adapter + .adapt_legacy_manifest(json) + .map_err(|err| ResolveError::AdaptError { + reference: reference.to_string(), + source: err, + })?; + + adapted.into_dependency_manifest() + } else { + // Raw dataset: parse directly into DependencyManifest + // Raw datasets don't have inline SQL, just schema information + serde_json::from_value(json.clone()).map_err(|err| ResolveError::ParseRawManifest { + reference: reference.to_string(), + source: err, + })? + }; + + // Cache the manifest JSON for future lookups + self.cache + .put(hash, &manifest) + .map_err(|err| ResolveError::CacheError { + hash: hash.clone(), + source: err, + })?; + + Ok(manifest) + } + + /// Resolves all dependencies and their transitive dependencies. + /// + /// This method: + /// 1. Resolves each direct dependency from the input map + /// 2. Recursively resolves transitive dependencies + /// 3. Detects cycles in the dependency graph + /// 4. Returns the complete dependency graph + /// + /// # Errors + /// + /// Returns an error if any dependency cannot be resolved or if a cycle + /// is detected. + pub async fn resolve_all( + &self, + dependencies: &BTreeMap, + ) -> Result { + let mut graph = DependencyGraph::new(); + let mut pending: VecDeque = dependencies.values().cloned().collect(); + let mut seen_hashes: BTreeSet = BTreeSet::new(); + + // Resolve direct dependencies first to build alias mapping + for (alias, reference) in dependencies { + let resolved = self.resolve_one(reference).await?; + graph.direct.insert(alias.clone(), resolved.hash.clone()); + pending.push_back(reference.clone()); + } + + // BFS to resolve all transitive dependencies + while let Some(reference) = pending.pop_front() { + let resolved = self.resolve_one(&reference).await?; + + if seen_hashes.contains(&resolved.hash) { + continue; + } + seen_hashes.insert(resolved.hash.clone()); + + // Collect hashes of this node's dependencies + let dep_hashes: Vec = { + let mut hashes = Vec::new(); + for dep_ref in resolved.manifest.dependencies.values() { + // Resolve each transitive dependency to get its hash + let dep_resolved = self.resolve_one(dep_ref).await?; + hashes.push(dep_resolved.hash.clone()); + + // Queue for further resolution if not seen + if !seen_hashes.contains(&dep_resolved.hash) { + pending.push_back(dep_ref.clone()); + } + } + hashes + }; + + graph.nodes.insert( + resolved.hash.clone(), + DependencyNode { + resolved, + deps: dep_hashes, + }, + ); + } + + // Verify no cycles using topological sort + graph.topological_order()?; + + Ok(graph) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, sync::Mutex}; + + use datasets_common::dataset_kind_str::DatasetKindStr; + + use super::*; + use crate::{ + canonical::hash_canonical, + lockfile::{LOCKFILE_VERSION, Lockfile, NodeInfo, RootInfo}, + }; + + /// Mock fetcher for testing that returns predefined manifests. + struct MockFetcher { + manifests: Mutex>, + by_hash: Mutex>, + } + + impl MockFetcher { + fn new() -> Self { + Self { + manifests: Mutex::new(BTreeMap::new()), + by_hash: Mutex::new(BTreeMap::new()), + } + } + + fn add_manifest(&self, reference: &str, manifest: serde_json::Value) { + self.manifests + .lock() + .unwrap() + .insert(reference.to_string(), manifest); + } + + #[allow(dead_code)] + fn add_by_hash(&self, hash: Hash, manifest: serde_json::Value) { + self.by_hash.lock().unwrap().insert(hash, manifest); + } + } + + #[async_trait] + impl ManifestFetcher for MockFetcher { + async fn fetch_by_version( + &self, + reference: &DepReference, + ) -> Result { + self.manifests + .lock() + .unwrap() + .get(&reference.to_string()) + .cloned() + .ok_or_else(|| ResolveError::NotFound { + reference: reference.to_string(), + }) + } + + async fn fetch_by_hash( + &self, + hash: &Hash, + ) -> Result, ResolveError> { + Ok(self.by_hash.lock().unwrap().get(hash).cloned()) + } + } + + fn create_minimal_manifest() -> serde_json::Value { + serde_json::json!({ + "kind": "manifest", + "dependencies": {}, + "tables": {}, + "functions": {} + }) + } + + /// Helper to create a test resolved dependency with default namespace/name. + fn create_test_resolved_dependency( + hash: Hash, + manifest: DependencyManifest, + ) -> ResolvedDependency { + ResolvedDependency { + hash, + manifest, + namespace: "test".parse().expect("valid namespace"), + name: "dataset".parse().expect("valid name"), + } + } + + #[allow(dead_code)] + fn create_manifest_with_deps(deps: BTreeMap<&str, &str>) -> serde_json::Value { + serde_json::json!({ + "kind": "manifest", + "dependencies": deps, + "tables": {}, + "functions": {} + }) + } + + #[test] + fn dependency_graph_new_creates_empty_graph() { + //* When + let graph = DependencyGraph::new(); + + //* Then + assert!(graph.is_empty()); + assert_eq!(graph.len(), 0); + assert!(graph.direct.is_empty()); + } + + #[test] + fn dependency_graph_default_creates_empty_graph() { + //* When + let graph = DependencyGraph::default(); + + //* Then + assert!(graph.is_empty()); + } + + #[tokio::test] + async fn resolve_one_with_version_fetches_and_caches() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let fetcher = MockFetcher::new(); + + let manifest = create_minimal_manifest(); + fetcher.add_manifest("test/dataset@1.0.0", manifest.clone()); + + let resolver = Resolver::new(fetcher, cache.clone()); + let reference: DepReference = "test/dataset@1.0.0".parse().expect("valid reference"); + + //* When + let result = resolver.resolve_one(&reference).await; + + //* Then + assert!(result.is_ok(), "resolution should succeed"); + let resolved = result.expect("should have resolved"); + + // Verify the manifest is now cached + assert!( + cache.contains(&resolved.hash), + "manifest should be cached after resolution" + ); + } + + #[tokio::test] + async fn resolve_one_with_missing_dependency_returns_not_found() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let fetcher = MockFetcher::new(); + // Don't add any manifests + + let resolver = Resolver::new(fetcher, cache); + let reference: DepReference = "missing/dataset@1.0.0".parse().expect("valid reference"); + + //* When + let result = resolver.resolve_one(&reference).await; + + //* Then + assert!(result.is_err(), "resolution should fail"); + let err = result.expect_err("should have error"); + assert!( + matches!(err, ResolveError::NotFound { .. }), + "should be NotFound error, got: {:?}", + err + ); + } + + #[tokio::test] + async fn resolve_one_accepts_raw_manifest_kind() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let fetcher = MockFetcher::new(); + + let manifest = serde_json::json!({ + "kind": "evm-rpc", + "network": "mainnet", + "start_block": 0, + "finalized_blocks_only": false, + "tables": { + "blocks": { + "schema": { "arrow": { "fields": [] } }, + "network": "mainnet" + } + } + }); + fetcher.add_manifest("test/raw@1.0.0", manifest); + + let resolver = Resolver::new(fetcher, cache); + let reference: DepReference = "test/raw@1.0.0".parse().expect("valid reference"); + + //* When + let result = resolver.resolve_one(&reference).await; + + //* Then + let resolved = result.expect("resolution should succeed"); + assert_eq!(resolved.manifest.kind.as_str(), "evm-rpc"); + assert!( + resolved + .manifest + .tables + .contains_key(&"blocks".parse().expect("valid table name")) + ); + } + + #[tokio::test] + async fn resolve_one_uses_cache_for_hash_reference() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + + // Pre-populate cache with a manifest + let manifest = DependencyManifest { + kind: DatasetKindStr::new("manifest".to_string()), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + }; + + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + cache.put(&hash, &manifest).expect("cache put should work"); + + let fetcher = MockFetcher::new(); + // Don't add anything to fetcher - should use cache + + let resolver = Resolver::new(fetcher, cache); + let reference: DepReference = format!("test/dataset@{}", hash) + .parse() + .expect("valid reference"); + + //* When + let result = resolver.resolve_one(&reference).await; + + //* Then + assert!(result.is_ok(), "resolution should succeed from cache"); + let resolved = result.expect("should have resolved"); + assert_eq!(resolved.hash, hash); + } + + #[tokio::test] + async fn resolve_one_offline_rejects_version_reference() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let fetcher = MockFetcher::new(); + let resolver = Resolver::new(fetcher, cache).with_offline(true); + let reference: DepReference = "test/dataset@1.0.0".parse().expect("valid reference"); + + //* When + let result = resolver.resolve_one(&reference).await; + + //* Then + let err = result.expect_err("should reject version refs in offline mode"); + assert!( + matches!(err, ResolveError::OfflineVersionReference(_)), + "expected OfflineVersionReference error, got: {err:?}" + ); + } + + #[tokio::test] + async fn resolve_one_offline_reports_cache_miss_for_hash_reference() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let fetcher = MockFetcher::new(); + let resolver = Resolver::new(fetcher, cache).with_offline(true); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + let reference: DepReference = format!("test/dataset@{}", hash) + .parse() + .expect("valid reference"); + + //* When + let result = resolver.resolve_one(&reference).await; + + //* Then + let err = result.expect_err("should report cache miss in offline mode"); + assert!( + matches!(err, ResolveError::OfflineCacheMiss(_)), + "expected OfflineCacheMiss error, got: {err:?}" + ); + } + + #[tokio::test] + async fn resolve_all_with_empty_dependencies_returns_empty_graph() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let fetcher = MockFetcher::new(); + let resolver = Resolver::new(fetcher, cache); + + let dependencies: BTreeMap = BTreeMap::new(); + + //* When + let result = resolver.resolve_all(&dependencies).await; + + //* Then + assert!(result.is_ok(), "resolution should succeed"); + let graph = result.expect("should have graph"); + assert!(graph.is_empty()); + assert!(graph.direct.is_empty()); + } + + #[tokio::test] + async fn resolve_all_resolves_single_dependency() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let fetcher = MockFetcher::new(); + + let manifest = create_minimal_manifest(); + fetcher.add_manifest("test/dataset@1.0.0", manifest); + + let resolver = Resolver::new(fetcher, cache); + + let alias: DepAlias = "test_dep".parse().expect("valid alias"); + let reference: DepReference = "test/dataset@1.0.0".parse().expect("valid reference"); + let mut dependencies = BTreeMap::new(); + dependencies.insert(alias.clone(), reference); + + //* When + let result = resolver.resolve_all(&dependencies).await; + + //* Then + assert!(result.is_ok(), "resolution should succeed"); + let graph = result.expect("should have graph"); + assert_eq!(graph.len(), 1); + assert!(graph.direct.contains_key(&alias)); + assert!(graph.get_by_alias(&alias).is_some()); + } + + #[tokio::test] + async fn resolve_all_locked_uses_lockfile_hashes() { + //* Given + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let fetcher = MockFetcher::new(); + + let manifest = create_minimal_manifest(); + let hash = hash_canonical(&manifest).expect("should compute hash"); + fetcher.add_by_hash(hash.clone(), manifest); + + let alias: DepAlias = "eth".parse().expect("valid alias"); + + let root = RootInfo { + namespace: "root".parse().expect("valid namespace"), + name: "dataset".parse().expect("valid name"), + version: "1.0.0".parse().expect("valid version"), + }; + + let mut dependencies = BTreeMap::new(); + dependencies.insert(alias.clone(), hash.clone()); + + let mut nodes = BTreeMap::new(); + nodes.insert( + hash.clone(), + NodeInfo { + namespace: "test".to_string(), + name: "dataset".to_string(), + deps: vec![], + }, + ); + + let lockfile = Lockfile { + version: LOCKFILE_VERSION, + root, + dependencies, + nodes, + }; + + let resolver = Resolver::new(fetcher, cache); + + //* When + let result = resolver.resolve_all_locked(&lockfile).await; + + //* Then + assert!(result.is_ok(), "resolution should succeed"); + let graph = result.expect("should have graph"); + assert_eq!(graph.direct.get(&alias), Some(&hash)); + let node = graph.get(&hash).expect("should have node"); + assert_eq!(node.resolved.hash, hash); + assert_eq!(node.resolved.namespace.as_str(), "test"); + assert_eq!(node.resolved.name.as_str(), "dataset"); + } + + #[test] + fn topological_order_with_empty_graph_returns_empty() { + //* Given + let graph = DependencyGraph::new(); + + //* When + let result = graph.topological_order(); + + //* Then + assert!(result.is_ok()); + let order = result.expect("should succeed"); + assert!(order.is_empty()); + } + + #[test] + fn topological_order_with_single_node_returns_that_node() { + //* Given + let mut graph = DependencyGraph::new(); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + let manifest = DependencyManifest { + kind: DatasetKindStr::new("manifest".to_string()), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + }; + + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash.clone(), manifest), + deps: vec![], + }, + ); + + //* When + let result = graph.topological_order(); + + //* Then + assert!(result.is_ok()); + let order = result.expect("should succeed"); + assert_eq!(order.len(), 1); + assert_eq!(order[0], hash); + } + + #[test] + fn topological_order_with_linear_deps_returns_correct_order() { + //* Given + let mut graph = DependencyGraph::new(); + + // Create three nodes: A -> B -> C (A depends on B, B depends on C) + let hash_a: Hash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .parse() + .expect("valid hash"); + let hash_b: Hash = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + .parse() + .expect("valid hash"); + let hash_c: Hash = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + .parse() + .expect("valid hash"); + + let manifest = DependencyManifest { + kind: DatasetKindStr::new("manifest".to_string()), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + }; + + graph.nodes.insert( + hash_a.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash_a.clone(), manifest.clone()), + deps: vec![hash_b.clone()], + }, + ); + + graph.nodes.insert( + hash_b.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash_b.clone(), manifest.clone()), + deps: vec![hash_c.clone()], + }, + ); + + graph.nodes.insert( + hash_c.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash_c.clone(), manifest), + deps: vec![], + }, + ); + + //* When + let result = graph.topological_order(); + + //* Then + assert!(result.is_ok()); + let order = result.expect("should succeed"); + assert_eq!(order.len(), 3); + + // C must come before B, B must come before A + let pos_a = order.iter().position(|h| h == &hash_a).unwrap(); + let pos_b = order.iter().position(|h| h == &hash_b).unwrap(); + let pos_c = order.iter().position(|h| h == &hash_c).unwrap(); + + assert!(pos_c < pos_b, "C should come before B"); + assert!(pos_b < pos_a, "B should come before A"); + } + + #[test] + fn topological_order_with_cycle_returns_error() { + //* Given + let mut graph = DependencyGraph::new(); + + // Create cycle: A -> B -> A + let hash_a: Hash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .parse() + .expect("valid hash"); + let hash_b: Hash = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + .parse() + .expect("valid hash"); + + let manifest = DependencyManifest { + kind: DatasetKindStr::new("manifest".to_string()), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + }; + + graph.nodes.insert( + hash_a.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash_a.clone(), manifest.clone()), + deps: vec![hash_b.clone()], + }, + ); + + graph.nodes.insert( + hash_b.clone(), + DependencyNode { + resolved: create_test_resolved_dependency(hash_b.clone(), manifest), + deps: vec![hash_a.clone()], // Creates cycle + }, + ); + + //* When + let result = graph.topological_order(); + + //* Then + assert!(result.is_err(), "should detect cycle"); + let err = result.expect_err("should have error"); + assert!( + matches!(err, ResolveError::CycleDetected { .. }), + "should be CycleDetected error, got: {:?}", + err + ); + } +} diff --git a/crates/core/dataset-authoring/src/schema.rs b/crates/core/dataset-authoring/src/schema.rs new file mode 100644 index 000000000..4bc1b726d --- /dev/null +++ b/crates/core/dataset-authoring/src/schema.rs @@ -0,0 +1,653 @@ +//! Schema inference via DataFusion. +//! +//! Infers Arrow schemas by planning CTAS SELECT statements against dependency +//! table schemas. This module provides the tooling to validate SQL queries and +//! extract their output schemas without executing them. +//! +//! # Overview +//! +//! During dataset authoring, we need to infer the output schema of each table +//! from its CTAS statement. This is done by: +//! +//! 1. Creating a planning context with dependency table schemas +//! 2. Planning the SELECT portion of the CTAS statement +//! 3. Extracting the schema from the logical plan +//! +//! # Example +//! +//! ```ignore +//! use dataset_authoring::schema::{SchemaContext, DependencyTable}; +//! use datafusion::arrow::datatypes::SchemaRef; +//! +//! // Create context with dependency tables +//! let mut ctx = SchemaContext::new(); +//! ctx.register_table(DependencyTable { +//! schema_name: "source".to_string(), +//! table_name: "transactions".parse().unwrap(), +//! schema: schema_ref, +//! }); +//! +//! // Infer schema from SELECT query - returns Arrow SchemaRef +//! let schema: SchemaRef = ctx.infer_schema(&query).await?; +//! ``` + +use std::sync::Arc; + +use async_trait::async_trait; +use datafusion::{ + arrow::datatypes::SchemaRef, + catalog::{MemorySchemaProvider, Session, TableProvider}, + common::DFSchemaRef, + datasource::TableType, + error::DataFusionError, + execution::{SessionStateBuilder, config::SessionConfig, context::SessionContext}, + logical_expr::{Expr, ScalarUDF}, + physical_plan::ExecutionPlan, + sql::sqlparser::ast, +}; +use datasets_common::table_name::TableName; + +/// A dependency table to register for schema inference. +/// +/// Represents a table from a dependency dataset that can be referenced +/// in SQL queries during schema inference. +#[derive(Debug, Clone)] +pub struct DependencyTable { + /// The schema name under which the table is registered (e.g., the dependency alias). + pub schema_name: String, + /// The table name within the schema. + pub table_name: TableName, + /// The Arrow schema of the table. + pub schema: SchemaRef, +} + +/// Context for inferring schemas from SQL queries. +/// +/// Holds dependency table schemas and provides methods to plan SQL +/// queries and extract their output schemas. +pub struct SchemaContext { + session_config: SessionConfig, + tables: Vec, + udfs: Vec, +} + +impl Default for SchemaContext { + fn default() -> Self { + Self::new() + } +} + +impl SchemaContext { + /// Creates a new empty schema context. + pub fn new() -> Self { + let session_config = SessionConfig::from_env().unwrap_or_default().set( + "datafusion.catalog.default_catalog", + &common::query_context::default_catalog_name(), + ); + + Self { + session_config, + tables: Vec::new(), + udfs: Vec::new(), + } + } + + /// Registers a dependency table for use in schema inference. + pub fn register_table(&mut self, table: DependencyTable) { + self.tables.push(table); + } + + /// Registers a custom UDF for schema inference. + pub fn register_udf(&mut self, udf: ScalarUDF) { + self.udfs.push(udf); + } + + /// Infers the output schema of a SELECT query. + /// + /// Plans the query against the registered dependency tables and + /// returns the Arrow schema of the query result. + /// + /// # Arguments + /// + /// * `query` - The SELECT query to plan (the inner query from a CTAS statement). + /// + /// # Returns + /// + /// * `Ok(SchemaRef)` - The inferred Arrow schema of the query result. + /// * `Err(SchemaInferenceError)` - If planning fails. + pub async fn infer_schema( + &self, + query: &ast::Query, + ) -> Result { + let ctx = self.create_session_context()?; + + // Create a SELECT statement from the query + let stmt = datafusion::sql::parser::Statement::Statement(Box::new(ast::Statement::Query( + Box::new(query.clone()), + ))); + + // Plan the statement to get the logical plan + let plan = ctx + .state() + .statement_to_plan(stmt) + .await + .map_err(SchemaInferenceError::Planning)?; + + // Extract schema from the logical plan - convert DFSchema to Arrow Schema + let df_schema: DFSchemaRef = plan.schema().clone(); + Ok(Arc::new(df_schema.as_arrow().clone())) + } + + /// Creates a DataFusion session context with all registered tables. + /// + /// This is useful when you need access to the session context for + /// additional validation (e.g., incremental constraint checking). + /// + /// # Errors + /// + /// Returns [`SchemaInferenceError::TableRegistration`] if table registration fails. + pub fn create_session_for_validation(&self) -> Result { + self.create_session_context() + } + + /// Creates a DataFusion session context with all registered tables. + fn create_session_context(&self) -> Result { + let state = SessionStateBuilder::new() + .with_config(self.session_config.clone()) + .with_runtime_env(Default::default()) + .with_default_features() + .build(); + + let ctx = SessionContext::new_with_state(state); + + // Register each dependency table + for table in &self.tables { + // Create the schema (namespace) if it doesn't exist + self.ensure_schema_exists(&ctx, &table.schema_name); + + // Create a schema-only table provider + let provider = Arc::new(SchemaOnlyTable { + schema: table.schema.clone(), + }); + + // Build the table reference + let table_ref = datafusion::sql::TableReference::partial( + table.schema_name.clone(), + table.table_name.as_str().to_string(), + ); + + ctx.register_table(table_ref, provider) + .map_err(SchemaInferenceError::TableRegistration)?; + } + + // Register built-in UDFs + self.register_udfs(&ctx); + + Ok(ctx) + } + + /// Ensures a schema (namespace) exists in the catalog. + fn ensure_schema_exists(&self, ctx: &SessionContext, schema_name: &str) { + let catalog_names = ctx.catalog_names(); + let catalog_name = catalog_names.first().expect("default catalog should exist"); + + if let Some(catalog) = ctx.catalog(catalog_name) + && catalog.schema(schema_name).is_none() + { + let schema_provider = Arc::new(MemorySchemaProvider::new()); + // Ignoring the result - if registration fails, table registration will fail later + let _ = catalog.register_schema(schema_name, schema_provider); + } + } + + /// Registers built-in UDFs for query planning. + fn register_udfs(&self, ctx: &SessionContext) { + // Register the same UDFs that are available in the runtime + for udf in common::query_context::udfs() { + ctx.register_udf(udf); + } + for udaf in common::query_context::udafs() { + ctx.register_udaf(udaf); + } + for udf in &self.udfs { + ctx.register_udf(udf.clone()); + } + } +} + +/// A minimal TableProvider that only provides schema information. +/// +/// Used for planning queries without execution. The `scan` method is +/// unreachable as this provider is only used for schema inference. +#[derive(Debug)] +struct SchemaOnlyTable { + schema: SchemaRef, +} + +#[async_trait] +impl TableProvider for SchemaOnlyTable { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + async fn scan( + &self, + _state: &dyn Session, + _projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> Result, DataFusionError> { + unreachable!("SchemaOnlyTable should never be scanned; it is only used for planning") + } +} + +/// Errors that can occur during schema inference. +#[derive(Debug, thiserror::Error)] +pub enum SchemaInferenceError { + /// Failed to plan the SQL query. + #[error("failed to plan SQL query")] + Planning(#[source] DataFusionError), + + /// Failed to register a table in the catalog. + #[error("failed to register table in catalog")] + TableRegistration(#[source] DataFusionError), +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + + use super::*; + + /// Creates a simple test schema with the given fields. + fn test_schema(fields: Vec<(&str, DataType, bool)>) -> SchemaRef { + Arc::new(Schema::new( + fields + .into_iter() + .map(|(name, dtype, nullable)| Field::new(name, dtype, nullable)) + .collect::>(), + )) + } + + mod infer_schema_tests { + use datafusion::sql::sqlparser::parser::Parser; + + use super::*; + + /// Helper to parse a SELECT query from SQL string. + fn parse_select_query(sql: &str) -> ast::Query { + let dialect = datafusion::sql::sqlparser::dialect::GenericDialect {}; + let mut statements = Parser::parse_sql(&dialect, sql).expect("SQL should parse"); + match statements.pop().expect("should have one statement") { + ast::Statement::Query(query) => *query, + other => panic!("expected SELECT query, got: {:?}", other), + } + } + + #[tokio::test] + async fn infers_simple_select_all() { + //* Given + let mut ctx = SchemaContext::new(); + ctx.register_table(DependencyTable { + schema_name: "source".to_string(), + table_name: "transactions".parse().unwrap(), + schema: test_schema(vec![ + ("block_num", DataType::UInt64, false), + ("tx_hash", DataType::Utf8, false), + ("value", DataType::UInt64, true), + ]), + }); + + let query = parse_select_query("SELECT * FROM source.transactions"); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + let schema = result.expect("schema inference should succeed"); + assert_eq!(schema.fields().len(), 3); + assert_eq!(schema.field(0).name(), "block_num"); + assert_eq!(schema.field(1).name(), "tx_hash"); + assert_eq!(schema.field(2).name(), "value"); + } + + #[tokio::test] + async fn infers_projected_columns() { + //* Given + let mut ctx = SchemaContext::new(); + ctx.register_table(DependencyTable { + schema_name: "source".to_string(), + table_name: "transactions".parse().unwrap(), + schema: test_schema(vec![ + ("block_num", DataType::UInt64, false), + ("tx_hash", DataType::Utf8, false), + ("value", DataType::UInt64, true), + ("gas_used", DataType::UInt64, true), + ]), + }); + + let query = parse_select_query("SELECT block_num, tx_hash FROM source.transactions"); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + let schema = result.expect("schema inference should succeed"); + assert_eq!(schema.fields().len(), 2); + assert_eq!(schema.field(0).name(), "block_num"); + assert_eq!(schema.field(1).name(), "tx_hash"); + } + + #[tokio::test] + async fn infers_aliased_columns() { + //* Given + let mut ctx = SchemaContext::new(); + ctx.register_table(DependencyTable { + schema_name: "source".to_string(), + table_name: "transactions".parse().unwrap(), + schema: test_schema(vec![ + ("block_num", DataType::UInt64, false), + ("value", DataType::UInt64, true), + ]), + }); + + let query = parse_select_query( + "SELECT block_num AS block, value AS amount FROM source.transactions", + ); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + let schema = result.expect("schema inference should succeed"); + assert_eq!(schema.fields().len(), 2); + assert_eq!(schema.field(0).name(), "block"); + assert_eq!(schema.field(1).name(), "amount"); + } + + #[tokio::test] + async fn infers_filtered_query() { + //* Given + let mut ctx = SchemaContext::new(); + ctx.register_table(DependencyTable { + schema_name: "source".to_string(), + table_name: "transactions".parse().unwrap(), + schema: test_schema(vec![ + ("block_num", DataType::UInt64, false), + ("value", DataType::UInt64, true), + ]), + }); + + let query = parse_select_query("SELECT * FROM source.transactions WHERE value > 1000"); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + let schema = result.expect("schema inference should succeed"); + assert_eq!(schema.fields().len(), 2); + } + + #[tokio::test] + async fn infers_join_schema() { + //* Given + let mut ctx = SchemaContext::new(); + ctx.register_table(DependencyTable { + schema_name: "source".to_string(), + table_name: "transactions".parse().unwrap(), + schema: test_schema(vec![ + ("block_num", DataType::UInt64, false), + ("tx_hash", DataType::Utf8, false), + ]), + }); + ctx.register_table(DependencyTable { + schema_name: "source".to_string(), + table_name: "blocks".parse().unwrap(), + schema: test_schema(vec![ + ("number", DataType::UInt64, false), + ("timestamp", DataType::UInt64, false), + ]), + }); + + let query = parse_select_query( + "SELECT t.tx_hash, b.timestamp \ + FROM source.transactions t \ + JOIN source.blocks b ON t.block_num = b.number", + ); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + let schema = result.expect("schema inference should succeed"); + assert_eq!(schema.fields().len(), 2); + assert_eq!(schema.field(0).name(), "tx_hash"); + assert_eq!(schema.field(1).name(), "timestamp"); + } + + #[tokio::test] + async fn infers_cte_schema() { + //* Given + let mut ctx = SchemaContext::new(); + ctx.register_table(DependencyTable { + schema_name: "source".to_string(), + table_name: "transactions".parse().unwrap(), + schema: test_schema(vec![ + ("block_num", DataType::UInt64, false), + ("value", DataType::UInt64, true), + ]), + }); + + let query = parse_select_query( + "WITH filtered AS (SELECT * FROM source.transactions WHERE value > 0) \ + SELECT block_num FROM filtered", + ); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + let schema = result.expect("schema inference should succeed"); + assert_eq!(schema.fields().len(), 1); + assert_eq!(schema.field(0).name(), "block_num"); + } + + #[tokio::test] + async fn preserves_nullability() { + //* Given + let mut ctx = SchemaContext::new(); + ctx.register_table(DependencyTable { + schema_name: "source".to_string(), + table_name: "data".parse().unwrap(), + schema: test_schema(vec![ + ("required_col", DataType::UInt64, false), + ("optional_col", DataType::UInt64, true), + ]), + }); + + let query = parse_select_query("SELECT required_col, optional_col FROM source.data"); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + let schema = result.expect("schema inference should succeed"); + assert!( + !schema.field(0).is_nullable(), + "required_col should not be nullable" + ); + assert!( + schema.field(1).is_nullable(), + "optional_col should be nullable" + ); + } + + #[tokio::test] + async fn infers_expression_types() { + //* Given + let mut ctx = SchemaContext::new(); + ctx.register_table(DependencyTable { + schema_name: "source".to_string(), + table_name: "data".parse().unwrap(), + schema: test_schema(vec![ + ("a", DataType::Int64, false), + ("b", DataType::Int64, false), + ]), + }); + + let query = + parse_select_query("SELECT a + b AS sum, a * 2 AS doubled FROM source.data"); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + let schema = result.expect("schema inference should succeed"); + assert_eq!(schema.fields().len(), 2); + assert_eq!(schema.field(0).name(), "sum"); + assert_eq!(schema.field(1).name(), "doubled"); + } + + #[tokio::test] + async fn fails_on_missing_table() { + //* Given + let ctx = SchemaContext::new(); // No tables registered + + let query = parse_select_query("SELECT * FROM missing.table"); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + assert!(result.is_err(), "should fail when table doesn't exist"); + let err = result.unwrap_err(); + assert!( + matches!(err, SchemaInferenceError::Planning(_)), + "expected Planning error, got: {:?}", + err + ); + } + + #[tokio::test] + async fn fails_on_missing_column() { + //* Given + let mut ctx = SchemaContext::new(); + ctx.register_table(DependencyTable { + schema_name: "source".to_string(), + table_name: "data".parse().unwrap(), + schema: test_schema(vec![("existing_col", DataType::UInt64, false)]), + }); + + let query = parse_select_query("SELECT nonexistent_col FROM source.data"); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + assert!(result.is_err(), "should fail when column doesn't exist"); + } + + #[tokio::test] + async fn handles_multiple_schemas() { + //* Given + let mut ctx = SchemaContext::new(); + ctx.register_table(DependencyTable { + schema_name: "eth".to_string(), + table_name: "blocks".parse().unwrap(), + schema: test_schema(vec![("number", DataType::UInt64, false)]), + }); + ctx.register_table(DependencyTable { + schema_name: "base".to_string(), + table_name: "blocks".parse().unwrap(), + schema: test_schema(vec![("block_number", DataType::UInt64, false)]), + }); + + let query = parse_select_query( + "SELECT e.number, b.block_number \ + FROM eth.blocks e \ + JOIN base.blocks b ON e.number = b.block_number", + ); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + let schema = result.expect("schema inference should succeed"); + assert_eq!(schema.fields().len(), 2); + assert_eq!(schema.field(0).name(), "number"); + assert_eq!(schema.field(1).name(), "block_number"); + } + + #[tokio::test] + async fn arrow_cast_produces_explicit_type() { + //* Given + let mut ctx = SchemaContext::new(); + ctx.register_table(DependencyTable { + schema_name: "source".to_string(), + table_name: "data".parse().unwrap(), + schema: test_schema(vec![ + ("int_col", DataType::Int64, false), + ("str_col", DataType::Utf8, false), + ]), + }); + + // Use arrow_cast to explicitly convert Int64 to UInt32 + let query = parse_select_query( + "SELECT arrow_cast(int_col, 'UInt32') AS casted_col FROM source.data", + ); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + let schema = result.expect("schema inference should succeed"); + assert_eq!(schema.fields().len(), 1); + assert_eq!(schema.field(0).name(), "casted_col"); + assert_eq!( + schema.field(0).data_type(), + &DataType::UInt32, + "arrow_cast should produce the explicitly specified type" + ); + } + + #[tokio::test] + async fn arrow_cast_works_with_complex_types() { + //* Given + let mut ctx = SchemaContext::new(); + ctx.register_table(DependencyTable { + schema_name: "source".to_string(), + table_name: "data".parse().unwrap(), + schema: test_schema(vec![("value", DataType::Int64, false)]), + }); + + // Cast to Decimal type with precision and scale + let query = parse_select_query( + "SELECT arrow_cast(value, 'Decimal128(18, 8)') AS decimal_col FROM source.data", + ); + + //* When + let result = ctx.infer_schema(&query).await; + + //* Then + let schema = result.expect("schema inference should succeed"); + assert_eq!(schema.fields().len(), 1); + assert_eq!(schema.field(0).name(), "decimal_col"); + // Verify the type is Decimal128 with the specified precision and scale + assert!( + matches!(schema.field(0).data_type(), DataType::Decimal128(18, 8)), + "arrow_cast should produce Decimal128(18, 8), got: {:?}", + schema.field(0).data_type() + ); + } + } +} diff --git a/crates/core/dataset-authoring/src/validation.rs b/crates/core/dataset-authoring/src/validation.rs new file mode 100644 index 000000000..7da5d41da --- /dev/null +++ b/crates/core/dataset-authoring/src/validation.rs @@ -0,0 +1,1471 @@ +//! Shared validation pipeline for dataset authoring. +//! +//! Performs configuration parsing, model discovery, dependency resolution, +//! SQL rendering, SELECT-only validation, incremental checks, and schema +//! inference. This pipeline is validation-only and does not write artifacts. + +use std::{ + any::Any, + collections::{BTreeMap, BTreeSet}, + fs, + path::{Path, PathBuf}, +}; + +use common::{ + incrementalizer::NonIncrementalQueryError, + plan_visitors::is_incremental, + query_context, + sql::{ + FunctionReference, ResolveFunctionReferencesError, ResolveTableReferencesError, + TableReference, parse, resolve_function_references, resolve_table_references, + }, +}; +use datafusion::{ + arrow::datatypes::{ + DataType as ArrowDataType, Field as ArrowField, Fields as ArrowFields, SchemaRef, + }, + common::utils::quote_identifier, + error::DataFusionError, + logical_expr::{ + ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, + }, +}; +use datasets_common::{manifest::DataType as ManifestDataType, table_name::TableName}; +use datasets_derived::{ + deps::{DepAlias, DepAliasError, DepAliasOrSelfRef, DepAliasOrSelfRefError, SELF_REF_KEYWORD}, + func_name::{ETH_CALL_FUNCTION_NAME, FuncName}, + sql_str::SqlStr, +}; + +use crate::{ + config::{self, AmpYaml}, + discovery::{DiscoveryError, discover_tables}, + jinja::{JinjaContext, RenderError, render_sql}, + lockfile::Lockfile, + manifest, + query::{SelectValidationError, validate_select}, + resolver::{DependencyGraph, ManifestFetcher, ResolveError, Resolver}, + schema::{DependencyTable, SchemaContext, SchemaInferenceError}, +}; + +/// Options for the validation pipeline. +pub struct ValidationOptions<'a, F> { + /// Project directory containing amp.yaml/amp.yml. + pub dir: &'a Path, + /// CLI variable overrides. + pub cli_vars: &'a [(String, String)], + /// Dependency resolver. + pub resolver: &'a Resolver, + /// Dependency resolution mode. + pub resolution: ResolutionMode<'a>, +} + +impl<'a, F> ValidationOptions<'a, F> { + /// Creates validation options with defaults. + pub fn new(dir: &'a Path, resolver: &'a Resolver) -> Self { + Self { + dir, + resolver, + cli_vars: &[], + resolution: ResolutionMode::Resolve, + } + } + + /// Provide CLI variable overrides. + pub fn with_cli_vars(mut self, cli_vars: &'a [(String, String)]) -> Self { + self.cli_vars = cli_vars; + self + } + + /// Configure dependency resolution mode. + pub fn with_resolution(mut self, resolution: ResolutionMode<'a>) -> Self { + self.resolution = resolution; + self + } +} + +/// Dependency resolution mode for validation. +#[derive(Debug, Clone, Copy)] +pub enum ResolutionMode<'a> { + /// Resolve dependencies from the registry (default). + Resolve, + /// Resolve dependencies using hashes from a lockfile. + Locked { lockfile: &'a Lockfile }, +} + +/// Output of the validation pipeline. +#[derive(Debug)] +pub struct ValidationOutput { + /// Resolved config file path. + pub config_path: PathBuf, + /// Parsed authoring configuration. + pub config: AmpYaml, + /// Discovered tables (table -> file path). + pub discovered_tables: BTreeMap, + /// Resolved dependency graph. + pub dependencies: DependencyGraph, + /// Rendered SQL and inferred schemas for each table. + pub validated_tables: BTreeMap, +} + +/// Validated table output. +#[derive(Debug, Clone)] +pub struct ValidatedTable { + /// Path to the source SQL file, relative to the project root. + pub source_path: PathBuf, + /// Rendered SQL after Jinja templating. + pub rendered_sql: String, + /// Inferred Arrow schema from the rendered SQL. + pub schema: SchemaRef, +} + +/// Errors that can occur during validation. +#[derive(Debug, thiserror::Error)] +pub enum ValidationError { + /// Failed to locate, read, or parse the authoring config. + #[error(transparent)] + ConfigLoad(#[from] config::ConfigLoadError), + + /// CLI vars not declared in amp.yaml. + #[error( + "unknown CLI vars: {unknown:?}. Declare them under `vars` in {}", + config_path.display() + )] + UnknownCliVars { + unknown: Vec, + config_path: PathBuf, + }, + + /// Model discovery failed. + #[error("model discovery failed")] + Discovery(#[source] DiscoveryError), + + /// Dependency resolution failed. + #[error("dependency resolution failed")] + Resolution(#[source] ResolveError), + + /// Failed to read SQL template file. + #[error("failed to read SQL file {}", path.display())] + ReadSqlFile { + path: PathBuf, + #[source] + source: std::io::Error, + }, + + /// Jinja template rendering failed. + #[error("failed to render SQL for table '{table}'")] + JinjaRender { + table: TableName, + #[source] + source: RenderError, + }, + + /// Invalid function data type. + #[error("invalid data type '{type_str}' for function '{func_name}'")] + InvalidFunctionType { + func_name: FuncName, + type_str: String, + #[source] + source: serde_json::Error, + }, + + /// Function source file is missing. + #[error("function source file not found for '{func_name}' at {}", path.display())] + FunctionSourceMissing { func_name: FuncName, path: PathBuf }, + + /// Failed to read function source file metadata. + #[error("failed to read function source file for '{func_name}' at {}", path.display())] + FunctionSourceRead { + func_name: FuncName, + path: PathBuf, + #[source] + source: std::io::Error, + }, + + /// Invalid SQL string. + #[error("invalid SQL string for table '{table}'")] + InvalidSqlString { table: TableName }, + + /// SQL parsing failed. + #[error("failed to parse SQL for table '{table}'")] + SqlParse { + table: TableName, + #[source] + source: common::sql::ParseSqlError, + }, + + /// Statement is not a SELECT query. + #[error("invalid SQL for model '{table}'")] + NotSelectStatement { + table: TableName, + #[source] + source: SelectValidationError, + }, + + /// Failed to resolve table references from SQL. + #[error("failed to resolve table references for table '{table}'")] + TableReferenceResolution { + table: TableName, + #[source] + source: ResolveTableReferencesError, + }, + + /// Catalog-qualified table reference detected in SQL. + #[error("catalog-qualified table reference in table '{table}'")] + CatalogQualifiedTableInSql { + table: TableName, + #[source] + source: ResolveTableReferencesError, + }, + + /// Invalid table name in SQL reference. + #[error("invalid table name in SQL for table '{table}'")] + InvalidTableName { + table: TableName, + #[source] + source: ResolveTableReferencesError, + }, + + /// Unqualified table reference detected. + #[error("unqualified table reference in table '{table}': {table_ref}")] + UnqualifiedTableReference { table: TableName, table_ref: String }, + + /// Dependency alias referenced in SQL was not declared. + #[error("dependency alias '{alias}' not found for table '{table}'")] + DependencyAliasNotFound { table: TableName, alias: DepAlias }, + + /// Referenced table not found in dependency manifest. + #[error("table '{referenced_table}' not found in dependency '{alias}' for table '{table}'")] + TableNotFoundInDependency { + table: TableName, + alias: DepAlias, + referenced_table: TableName, + }, + + /// Failed to resolve function references from SQL. + #[error("failed to resolve function references for table '{table}'")] + FunctionReferenceResolution { + table: TableName, + #[source] + source: ResolveFunctionReferencesError, + }, + + /// Referenced function not found in dependency manifest. + #[error("function '{function}' not found in dependency '{alias}' for table '{table}'")] + FunctionNotFoundInDependency { + table: TableName, + alias: DepAlias, + function: FuncName, + }, + + /// Self-referenced function not defined in amp.yaml. + #[error("self-referenced function '{function}' not found for table '{table}'")] + SelfFunctionNotFound { + table: TableName, + function: FuncName, + }, + + /// eth_call is not available for the referenced dependency. + #[error("eth_call not available for dependency '{alias}' in table '{table}'")] + EthCallNotAvailable { table: TableName, alias: DepAlias }, + + /// SQL planning failed. + #[error("failed to plan SQL for table '{table}'")] + SqlPlanning { + table: TableName, + #[source] + source: query_context::Error, + }, + + /// Schema inference failed. + #[error("schema inference failed for table '{table}'")] + SchemaInference { + table: TableName, + #[source] + source: SchemaInferenceError, + }, + + /// Network inference failed. + #[error("network inference failed for table '{table}'")] + NetworkInference { + table: TableName, + #[source] + source: crate::manifest::ManifestError, + }, + + /// Query contains non-incremental operations. + #[error("query for table '{table}' contains non-incremental operations")] + NonIncremental { + table: TableName, + #[source] + source: NonIncrementalQueryError, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct StubUdf { + name: String, + signature: Signature, + return_type: ArrowDataType, +} + +impl StubUdf { + fn new(name: String, signature: Signature, return_type: ArrowDataType) -> Self { + Self { + name, + signature, + return_type, + } + } + + fn from_exact_signature( + name: String, + input_types: Vec, + return_type: ArrowDataType, + ) -> Self { + let signature = Signature { + type_signature: TypeSignature::Exact(input_types), + volatility: Volatility::Immutable, + parameter_names: None, + }; + Self::new(name, signature, return_type) + } + + fn eth_call(schema: &str) -> Self { + let name = format!("{}.{}", quote_identifier(schema), ETH_CALL_FUNCTION_NAME); + let signature = Signature { + type_signature: TypeSignature::Any(4), + volatility: Volatility::Volatile, + parameter_names: Some(vec![ + "from".to_string(), + "to".to_string(), + "input_data".to_string(), + "block".to_string(), + ]), + }; + let fields = ArrowFields::from_iter([ + ArrowField::new("data", ArrowDataType::Binary, true), + ArrowField::new("message", ArrowDataType::Utf8, true), + ]); + let return_type = ArrowDataType::Struct(fields); + Self::new(name, signature, return_type) + } +} + +impl ScalarUDFImpl for StubUdf { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + &self.name + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[ArrowDataType]) -> Result { + Ok(self.return_type.clone()) + } + + fn invoke_with_args( + &self, + _args: ScalarFunctionArgs, + ) -> Result { + Err(DataFusionError::Plan(format!( + "UDF '{}' is not executable during validation", + self.name + ))) + } +} + +fn qualified_udf_name(schema: &str, function: &FuncName) -> String { + format!("{}.{}", quote_identifier(schema), function) +} + +/// Run the shared validation pipeline for a dataset project. +/// +/// This function performs configuration parsing, model discovery, +/// dependency resolution, SQL rendering, SELECT validation, incremental +/// checks, and schema inference. It does not write any output artifacts. +/// +/// # Errors +/// +/// Returns [`ValidationError`] for any validation failures. +pub async fn validate_project( + options: ValidationOptions<'_, F>, +) -> Result { + let (config_path, config) = config::load_from_dir(options.dir)?; + validate_project_with_config(options, config_path, config).await +} + +/// Run the shared validation pipeline for a dataset project with a pre-loaded config. +/// +/// This function performs model discovery, dependency resolution, SQL rendering, +/// SELECT validation, incremental checks, and schema inference. It does not write +/// any output artifacts. +/// +/// # Errors +/// +/// Returns [`ValidationError`] for any validation failures. +pub async fn validate_project_with_config( + options: ValidationOptions<'_, F>, + config_path: PathBuf, + config: AmpYaml, +) -> Result { + let ValidationOptions { + dir, + cli_vars, + resolver, + resolution, + } = options; + + validate_cli_vars(&config_path, &config, cli_vars)?; + validate_function_types(&config)?; + validate_function_sources(dir, &config)?; + + let discovered_tables = + discover_tables(dir, &config.tables).map_err(ValidationError::Discovery)?; + + let dependencies = match resolution { + ResolutionMode::Resolve => resolver + .resolve_all(&config.dependencies) + .await + .map_err(ValidationError::Resolution)?, + ResolutionMode::Locked { lockfile } => resolver + .resolve_all_locked(lockfile) + .await + .map_err(ValidationError::Resolution)?, + }; + + let jinja_ctx = build_jinja_context(&config, cli_vars); + + let mut schema_ctx = SchemaContext::new(); + register_dependency_tables(&mut schema_ctx, &dependencies); + register_dependency_udfs(&mut schema_ctx, &dependencies)?; + register_self_udfs(&mut schema_ctx, &config)?; + + let mut validated_tables = BTreeMap::new(); + for (table_name, sql_path) in &discovered_tables { + let validated = validate_table( + table_name, + sql_path, + dir, + &jinja_ctx, + &schema_ctx, + &config, + &dependencies, + ) + .await?; + validated_tables.insert(table_name.clone(), validated); + } + + validate_network_inference(&discovered_tables, &dependencies)?; + + Ok(ValidationOutput { + config_path, + config, + discovered_tables, + dependencies, + validated_tables, + }) +} + +fn validate_cli_vars( + config_path: &Path, + config: &AmpYaml, + cli_vars: &[(String, String)], +) -> Result<(), ValidationError> { + if cli_vars.is_empty() { + return Ok(()); + } + + let allowed: BTreeSet<&str> = match &config.vars { + Some(vars) => vars.keys().map(String::as_str).collect(), + None => BTreeSet::new(), + }; + + let mut unknown = BTreeSet::new(); + for (name, _) in cli_vars { + if !allowed.contains(name.as_str()) { + unknown.insert(name.clone()); + } + } + + if unknown.is_empty() { + Ok(()) + } else { + Err(ValidationError::UnknownCliVars { + unknown: unknown.into_iter().collect(), + config_path: config_path.to_path_buf(), + }) + } +} + +fn validate_function_sources(project_dir: &Path, config: &AmpYaml) -> Result<(), ValidationError> { + let Some(functions) = &config.functions else { + return Ok(()); + }; + + for (func_name, func_def) in functions { + let path = project_dir.join(&func_def.source); + match fs::metadata(&path) { + Ok(metadata) => { + if !metadata.is_file() { + return Err(ValidationError::FunctionSourceMissing { + func_name: func_name.clone(), + path, + }); + } + fs::File::open(&path).map_err(|source| ValidationError::FunctionSourceRead { + func_name: func_name.clone(), + path, + source, + })?; + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Err(ValidationError::FunctionSourceMissing { + func_name: func_name.clone(), + path, + }); + } + Err(err) => { + return Err(ValidationError::FunctionSourceRead { + func_name: func_name.clone(), + path, + source: err, + }); + } + } + } + + Ok(()) +} + +fn validate_function_types(config: &AmpYaml) -> Result<(), ValidationError> { + let Some(functions) = &config.functions else { + return Ok(()); + }; + + for (func_name, func_def) in functions { + for type_str in &func_def.input_types { + let _ = parse_function_type(func_name, type_str)?; + } + + let _ = parse_function_type(func_name, &func_def.output_type)?; + } + + Ok(()) +} + +fn parse_function_type( + func_name: &FuncName, + type_str: &str, +) -> Result { + let value = serde_json::Value::String(type_str.to_string()); + serde_json::from_value(value).map_err(|source| ValidationError::InvalidFunctionType { + func_name: func_name.clone(), + type_str: type_str.to_string(), + source, + }) +} + +fn parse_function_signature( + func_name: &FuncName, + func_def: &config::FunctionDef, +) -> Result<(Vec, ArrowDataType), ValidationError> { + let mut inputs = Vec::with_capacity(func_def.input_types.len()); + for type_str in &func_def.input_types { + let data_type = parse_function_type(func_name, type_str)?; + inputs.push(data_type.into_arrow()); + } + + let output = parse_function_type(func_name, &func_def.output_type)?.into_arrow(); + + Ok((inputs, output)) +} + +fn validate_network_inference( + discovered_tables: &BTreeMap, + dependencies: &DependencyGraph, +) -> Result<(), ValidationError> { + for table_name in discovered_tables.keys() { + manifest::default_network_resolver(table_name, dependencies).map_err(|source| { + ValidationError::NetworkInference { + table: table_name.clone(), + source, + } + })?; + } + + Ok(()) +} + +fn build_jinja_context(config: &AmpYaml, cli_vars: &[(String, String)]) -> JinjaContext { + let yaml_vars = config.vars.clone().unwrap_or_default(); + JinjaContext::builder() + .with_cli_vars(cli_vars.iter().cloned()) + .with_yaml_vars(yaml_vars) + .with_dep_aliases(config.dependencies.keys().cloned()) + .build() +} + +fn register_dependency_tables(ctx: &mut SchemaContext, dep_graph: &DependencyGraph) { + for (alias, hash) in &dep_graph.direct { + if let Some(node) = dep_graph.get(hash) { + for (table_name, table) in &node.resolved.manifest.tables { + let arrow_schema = table.schema.arrow.clone().into_schema_ref(); + ctx.register_table(DependencyTable { + schema_name: alias.to_string(), + table_name: table_name.clone(), + schema: arrow_schema, + }); + } + } + } +} + +fn register_dependency_udfs( + ctx: &mut SchemaContext, + dep_graph: &DependencyGraph, +) -> Result<(), ValidationError> { + for (alias, hash) in &dep_graph.direct { + let Some(node) = dep_graph.get(hash) else { + continue; + }; + let manifest = &node.resolved.manifest; + + for (func_name, func_def) in &manifest.functions { + let input_types = func_def + .input_types + .iter() + .cloned() + .map(ManifestDataType::into_arrow) + .collect(); + let output_type = func_def.output_type.clone().into_arrow(); + let udf_name = qualified_udf_name(alias.as_str(), func_name); + let udf = StubUdf::from_exact_signature(udf_name, input_types, output_type); + ctx.register_udf(udf.into()); + } + + if manifest.kind.as_str() == "evm-rpc" { + let udf = StubUdf::eth_call(alias.as_str()); + ctx.register_udf(udf.into()); + } + } + + Ok(()) +} + +fn register_self_udfs(ctx: &mut SchemaContext, config: &AmpYaml) -> Result<(), ValidationError> { + let Some(functions) = &config.functions else { + return Ok(()); + }; + + for (func_name, func_def) in functions { + let (inputs, output) = parse_function_signature(func_name, func_def)?; + let udf_name = qualified_udf_name(SELF_REF_KEYWORD, func_name); + let udf = StubUdf::from_exact_signature(udf_name, inputs, output); + ctx.register_udf(udf.into()); + } + + Ok(()) +} + +fn validate_table_references( + table_name: &TableName, + stmt: &datafusion::sql::parser::Statement, + config: &AmpYaml, + dependencies: &DependencyGraph, +) -> Result<(), ValidationError> { + let table_refs = resolve_table_references::(stmt).map_err(|err| match &err { + ResolveTableReferencesError::InvalidTableName { .. } => ValidationError::InvalidTableName { + table: table_name.clone(), + source: err, + }, + ResolveTableReferencesError::CatalogQualifiedTable { .. } => { + ValidationError::CatalogQualifiedTableInSql { + table: table_name.clone(), + source: err, + } + } + _ => ValidationError::TableReferenceResolution { + table: table_name.clone(), + source: err, + }, + })?; + + for table_ref in table_refs { + match table_ref { + TableReference::Bare { .. } => { + return Err(ValidationError::UnqualifiedTableReference { + table: table_name.clone(), + table_ref: table_ref.to_quoted_string(), + }); + } + TableReference::Partial { schema, table } => { + let alias = schema.as_ref(); + let Some(hash) = dependencies.direct.get(alias) else { + return Err(ValidationError::DependencyAliasNotFound { + table: table_name.clone(), + alias: alias.clone(), + }); + }; + + let Some(node) = dependencies.get(hash) else { + return Err(ValidationError::DependencyAliasNotFound { + table: table_name.clone(), + alias: alias.clone(), + }); + }; + + if !node.resolved.manifest.tables.contains_key(table.as_ref()) { + return Err(ValidationError::TableNotFoundInDependency { + table: table_name.clone(), + alias: alias.clone(), + referenced_table: table.as_ref().clone(), + }); + } + } + } + } + + let func_refs = resolve_function_references::(stmt).map_err(|err| { + ValidationError::FunctionReferenceResolution { + table: table_name.clone(), + source: err, + } + })?; + + for func_ref in func_refs { + match func_ref { + FunctionReference::Bare { .. } => continue, + FunctionReference::Qualified { schema, function } => match schema.as_ref() { + DepAliasOrSelfRef::SelfRef => { + let Some(functions) = &config.functions else { + return Err(ValidationError::SelfFunctionNotFound { + table: table_name.clone(), + function: function.as_ref().clone(), + }); + }; + + if !functions.contains_key(function.as_ref()) { + return Err(ValidationError::SelfFunctionNotFound { + table: table_name.clone(), + function: function.as_ref().clone(), + }); + } + } + DepAliasOrSelfRef::DepAlias(alias) => { + let Some(hash) = dependencies.direct.get(alias) else { + return Err(ValidationError::DependencyAliasNotFound { + table: table_name.clone(), + alias: alias.clone(), + }); + }; + + let Some(node) = dependencies.get(hash) else { + return Err(ValidationError::DependencyAliasNotFound { + table: table_name.clone(), + alias: alias.clone(), + }); + }; + + if function.as_ref() == ETH_CALL_FUNCTION_NAME { + if node.resolved.manifest.kind.as_str() != "evm-rpc" { + return Err(ValidationError::EthCallNotAvailable { + table: table_name.clone(), + alias: alias.clone(), + }); + } + continue; + } + + if !node + .resolved + .manifest + .functions + .contains_key(function.as_ref()) + { + return Err(ValidationError::FunctionNotFoundInDependency { + table: table_name.clone(), + alias: alias.clone(), + function: function.as_ref().clone(), + }); + } + } + }, + } + } + + Ok(()) +} + +async fn validate_table( + table_name: &TableName, + sql_path: &Path, + project_dir: &Path, + jinja_ctx: &JinjaContext, + schema_ctx: &SchemaContext, + config: &AmpYaml, + dependencies: &DependencyGraph, +) -> Result { + let template_path = project_dir.join(sql_path); + let template = + fs::read_to_string(&template_path).map_err(|source| ValidationError::ReadSqlFile { + path: template_path.clone(), + source, + })?; + + let rendered_sql = render_sql(&template, table_name.as_str(), jinja_ctx).map_err(|source| { + ValidationError::JinjaRender { + table: table_name.clone(), + source, + } + })?; + + let sql_str: SqlStr = rendered_sql + .parse() + .map_err(|_| ValidationError::InvalidSqlString { + table: table_name.clone(), + })?; + let stmt = parse(&sql_str).map_err(|source| ValidationError::SqlParse { + table: table_name.clone(), + source, + })?; + + let select_info = validate_select(&stmt, table_name.clone()).map_err(|source| { + ValidationError::NotSelectStatement { + table: table_name.clone(), + source, + } + })?; + + validate_table_references(table_name, &stmt, config, dependencies)?; + + let schema = schema_ctx + .infer_schema(&select_info.query) + .await + .map_err(|source| ValidationError::SchemaInference { + table: table_name.clone(), + source, + })?; + + validate_incremental_constraints(&select_info.query, schema_ctx, table_name).await?; + + Ok(ValidatedTable { + source_path: sql_path.to_path_buf(), + rendered_sql, + schema, + }) +} + +async fn validate_incremental_constraints( + query: &datafusion::sql::sqlparser::ast::Query, + schema_ctx: &SchemaContext, + table_name: &TableName, +) -> Result<(), ValidationError> { + let ctx = schema_ctx + .create_session_for_validation() + .map_err(|source| ValidationError::SchemaInference { + table: table_name.clone(), + source, + })?; + + let stmt = datafusion::sql::parser::Statement::Statement(Box::new( + datafusion::sql::sqlparser::ast::Statement::Query(Box::new(query.clone())), + )); + + let plan = query_context::sql_to_plan(&ctx, stmt) + .await + .map_err(|source| ValidationError::SqlPlanning { + table: table_name.clone(), + source, + })?; + + is_incremental(&plan).map_err(|source| ValidationError::NonIncremental { + table: table_name.clone(), + source, + })?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; + + use datafusion::arrow::datatypes::DataType as ArrowDataType; + use datasets_common::{ + dataset_kind_str::DatasetKindStr, + hash::Hash, + manifest::{ArrowSchema, Field, Function, FunctionSource, TableSchema}, + table_name::TableName, + }; + use datasets_derived::{deps::DepAlias, sql_str::SqlStr}; + + use super::{ + AmpYaml, ValidationError, validate_cli_vars, validate_incremental_constraints, + validate_table_references, + }; + use crate::{ + dependency_manifest::{DependencyManifest, DependencyTable}, + resolver::{DependencyGraph, DependencyNode, ResolvedDependency}, + schema::SchemaContext, + }; + + fn parse_statement(sql: &str) -> datafusion::sql::parser::Statement { + let sql: SqlStr = sql.parse().expect("valid SQL string"); + common::sql::parse(&sql).expect("should parse SQL") + } + + fn make_table_schema() -> TableSchema { + TableSchema { + arrow: ArrowSchema { + fields: vec![Field { + name: "id".to_string(), + type_: ArrowDataType::Int64.into(), + nullable: false, + }], + }, + } + } + + fn make_dependency_manifest( + kind: &str, + tables: &[&str], + functions: &[&str], + ) -> DependencyManifest { + let mut table_map = BTreeMap::new(); + for table in tables { + table_map.insert( + table.parse().expect("valid table name"), + DependencyTable { + schema: make_table_schema(), + network: "mainnet".parse().expect("valid network"), + }, + ); + } + + let mut function_map = BTreeMap::new(); + for function in functions { + function_map.insert( + function.parse().expect("valid function name"), + Function { + input_types: vec![ArrowDataType::Int64.into()], + output_type: ArrowDataType::Int64.into(), + source: FunctionSource { + source: Arc::from("function stub() {}"), + filename: format!("{function}.js"), + }, + }, + ); + } + + DependencyManifest { + kind: DatasetKindStr::new(kind.to_string()), + dependencies: BTreeMap::new(), + tables: table_map, + functions: function_map, + } + } + + fn make_dependency_graph(alias: &str, manifest: DependencyManifest) -> DependencyGraph { + let mut graph = DependencyGraph::new(); + let alias: DepAlias = alias.parse().expect("valid alias"); + let hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + graph.direct.insert(alias.clone(), hash.clone()); + graph.nodes.insert( + hash.clone(), + DependencyNode { + resolved: ResolvedDependency { + hash, + manifest, + namespace: "test".parse().expect("valid namespace"), + name: "dataset".parse().expect("valid name"), + }, + deps: vec![], + }, + ); + + graph + } + + fn minimal_config() -> AmpYaml { + let yaml = r#" +amp: 1.0.0 +namespace: test_ns +name: test_dataset +version: 1.0.0 +tables: tables +"#; + AmpYaml::from_yaml(yaml).expect("should parse config") + } + + fn config_with_functions(functions: &[&str]) -> AmpYaml { + let mut yaml = String::from( + "amp: 1.0.0\nnamespace: test_ns\nname: test_dataset\nversion: 1.0.0\ntables: tables\n", + ); + + if !functions.is_empty() { + yaml.push_str("functions:\n"); + for func in functions { + yaml.push_str(&format!( + " {func}:\n input_types: [Int64]\n output_type: Int64\n source: functions/{func}.js\n" + )); + } + } + + AmpYaml::from_yaml(&yaml).expect("should parse config") + } + + #[test] + fn validate_cli_vars_rejects_unknown_keys() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: test_ns +name: test_dataset +version: 1.0.0 +vars: + network: mainnet +"#; + let config = AmpYaml::from_yaml(yaml).expect("should parse config"); + let config_path = PathBuf::from("amp.yaml"); + let cli_vars = vec![ + ("network".to_string(), "testnet".to_string()), + ("chain_id".to_string(), "1".to_string()), + ]; + + //* When + let result = validate_cli_vars(&config_path, &config, &cli_vars); + + //* Then + let err = result.expect_err("should reject unknown vars"); + match err { + ValidationError::UnknownCliVars { + unknown, + config_path: path, + } => { + assert_eq!(path, config_path); + assert_eq!(unknown, vec!["chain_id".to_string()]); + } + other => panic!("unexpected error: {other:?}"), + } + } + + #[test] + fn validate_cli_vars_accepts_known_keys() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: test_ns +name: test_dataset +version: 1.0.0 +vars: + network: mainnet + chain_id: "1" +"#; + let config = AmpYaml::from_yaml(yaml).expect("should parse config"); + let config_path = PathBuf::from("amp.yaml"); + let cli_vars = vec![ + ("network".to_string(), "testnet".to_string()), + ("chain_id".to_string(), "10".to_string()), + ]; + + //* When + let result = validate_cli_vars(&config_path, &config, &cli_vars); + + //* Then + assert!(result.is_ok(), "should accept declared vars"); + } + + mod function_sources { + use std::fs; + + use tempfile::TempDir; + + use super::{AmpYaml, ValidationError}; + use crate::validation::validate_function_sources; + + #[test] + fn accepts_existing_function_sources() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let functions_dir = dir.path().join("functions"); + fs::create_dir_all(&functions_dir).expect("should create functions dir"); + fs::write( + functions_dir.join("my_func.js"), + "function myFunc(x) { return x; }", + ) + .expect("should write function"); + + let yaml = r#" +amp: 1.0.0 +namespace: test_ns +name: test_dataset +version: 1.0.0 +tables: tables +functions: + myFunc: + input_types: [Int64] + output_type: Int64 + source: functions/my_func.js +"#; + + let config = AmpYaml::from_yaml(yaml).expect("should parse config"); + + //* When + let result = validate_function_sources(dir.path(), &config); + + //* Then + assert!(result.is_ok(), "function source should be accepted"); + } + + #[test] + fn rejects_missing_function_source() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let yaml = r#" +amp: 1.0.0 +namespace: test_ns +name: test_dataset +version: 1.0.0 +tables: tables +functions: + missingFunc: + input_types: [Utf8] + output_type: Utf8 + source: functions/missing.js +"#; + + let config = AmpYaml::from_yaml(yaml).expect("should parse config"); + + //* When + let result = validate_function_sources(dir.path(), &config); + + //* Then + let err = result.expect_err("should reject missing function source"); + assert!(matches!(err, ValidationError::FunctionSourceMissing { .. })); + } + } + + mod function_types { + use super::{AmpYaml, ValidationError}; + use crate::validation::validate_function_types; + + #[test] + fn validate_function_types_rejects_invalid_input_type() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: test_ns +name: test_dataset +version: 1.0.0 +tables: tables +functions: + myFunc: + input_types: [NotAType] + output_type: Utf8 + source: functions/my_func.js +"#; + let config = AmpYaml::from_yaml(yaml).expect("should parse config"); + + //* When + let result = validate_function_types(&config); + + //* Then + let err = result.expect_err("should reject invalid input type"); + match err { + ValidationError::InvalidFunctionType { type_str, .. } => { + assert_eq!(type_str, "NotAType"); + } + other => panic!("unexpected error: {other:?}"), + } + } + + #[test] + fn validate_function_types_rejects_invalid_output_type() { + //* Given + let yaml = r#" +amp: 1.0.0 +namespace: test_ns +name: test_dataset +version: 1.0.0 +tables: tables +functions: + myFunc: + input_types: [Utf8] + output_type: NotAType + source: functions/my_func.js +"#; + let config = AmpYaml::from_yaml(yaml).expect("should parse config"); + + //* When + let result = validate_function_types(&config); + + //* Then + let err = result.expect_err("should reject invalid output type"); + match err { + ValidationError::InvalidFunctionType { type_str, .. } => { + assert_eq!(type_str, "NotAType"); + } + other => panic!("unexpected error: {other:?}"), + } + } + } + + mod reference_validation { + use super::*; + + #[test] + fn validate_table_references_rejects_unqualified_table_reference() { + //* Given + let config = minimal_config(); + let manifest = make_dependency_manifest("evm-rpc", &["blocks"], &[]); + let dep_graph = make_dependency_graph("eth", manifest); + let stmt = parse_statement("SELECT * FROM blocks"); + let table: TableName = "my_table".parse().expect("valid table name"); + + //* When + let result = validate_table_references(&table, &stmt, &config, &dep_graph); + + //* Then + let err = result.expect_err("should reject unqualified table reference"); + assert!(matches!( + err, + ValidationError::UnqualifiedTableReference { .. } + )); + } + + #[test] + fn validate_table_references_rejects_catalog_qualified_table_reference() { + //* Given + let config = minimal_config(); + let manifest = make_dependency_manifest("evm-rpc", &["blocks"], &[]); + let dep_graph = make_dependency_graph("eth", manifest); + let stmt = parse_statement("SELECT * FROM catalog.schema.table"); + let table: TableName = "my_table".parse().expect("valid table name"); + + //* When + let result = validate_table_references(&table, &stmt, &config, &dep_graph); + + //* Then + let err = result.expect_err("should reject catalog-qualified table reference"); + assert!(matches!( + err, + ValidationError::CatalogQualifiedTableInSql { .. } + )); + } + + #[test] + fn validate_table_references_rejects_unknown_dependency_alias() { + //* Given + let config = minimal_config(); + let manifest = make_dependency_manifest("evm-rpc", &["blocks"], &[]); + let dep_graph = make_dependency_graph("eth", manifest); + let stmt = parse_statement("SELECT * FROM missing.blocks"); + let table: TableName = "my_table".parse().expect("valid table name"); + + //* When + let result = validate_table_references(&table, &stmt, &config, &dep_graph); + + //* Then + let err = result.expect_err("should reject unknown dependency alias"); + assert!(matches!( + err, + ValidationError::DependencyAliasNotFound { .. } + )); + } + + #[test] + fn validate_table_references_rejects_missing_dependency_table() { + //* Given + let config = minimal_config(); + let manifest = make_dependency_manifest("evm-rpc", &["blocks"], &[]); + let dep_graph = make_dependency_graph("eth", manifest); + let stmt = parse_statement("SELECT * FROM eth.missing_table"); + let table: TableName = "my_table".parse().expect("valid table name"); + + //* When + let result = validate_table_references(&table, &stmt, &config, &dep_graph); + + //* Then + let err = result.expect_err("should reject missing dependency table"); + assert!(matches!( + err, + ValidationError::TableNotFoundInDependency { .. } + )); + } + + #[test] + fn validate_table_references_accepts_self_function_reference() { + //* Given + let config = config_with_functions(&["myFunc"]); + let dep_graph = DependencyGraph::new(); + let stmt = parse_statement("SELECT self.myFunc(1)"); + let table: TableName = "my_table".parse().expect("valid table name"); + + //* When + let result = validate_table_references(&table, &stmt, &config, &dep_graph); + + //* Then + assert!(result.is_ok(), "self function reference should be accepted"); + } + + #[test] + fn validate_table_references_rejects_missing_self_function() { + //* Given + let config = minimal_config(); + let dep_graph = DependencyGraph::new(); + let stmt = parse_statement("SELECT self.missingFunc(1)"); + let table: TableName = "my_table".parse().expect("valid table name"); + + //* When + let result = validate_table_references(&table, &stmt, &config, &dep_graph); + + //* Then + let err = result.expect_err("should reject missing self function"); + assert!(matches!(err, ValidationError::SelfFunctionNotFound { .. })); + } + + #[test] + fn validate_table_references_accepts_dependency_function_reference() { + //* Given + let config = minimal_config(); + let manifest = make_dependency_manifest("manifest", &["blocks"], &["depFunc"]); + let dep_graph = make_dependency_graph("dep", manifest); + let stmt = parse_statement("SELECT dep.depFunc(1)"); + let table: TableName = "my_table".parse().expect("valid table name"); + + //* When + let result = validate_table_references(&table, &stmt, &config, &dep_graph); + + //* Then + assert!( + result.is_ok(), + "dependency function reference should be accepted" + ); + } + + #[test] + fn validate_table_references_rejects_missing_dependency_function() { + //* Given + let config = minimal_config(); + let manifest = make_dependency_manifest("manifest", &["blocks"], &[]); + let dep_graph = make_dependency_graph("dep", manifest); + let stmt = parse_statement("SELECT dep.missingFunc(1)"); + let table: TableName = "my_table".parse().expect("valid table name"); + + //* When + let result = validate_table_references(&table, &stmt, &config, &dep_graph); + + //* Then + let err = result.expect_err("should reject missing dependency function"); + assert!(matches!( + err, + ValidationError::FunctionNotFoundInDependency { .. } + )); + } + + #[test] + fn validate_table_references_accepts_eth_call_for_evm_rpc_dependency() { + //* Given + let config = minimal_config(); + let manifest = make_dependency_manifest("evm-rpc", &["blocks"], &[]); + let dep_graph = make_dependency_graph("dep", manifest); + let stmt = parse_statement("SELECT dep.eth_call(NULL, NULL, NULL, 'latest') AS result"); + let table: TableName = "my_table".parse().expect("valid table name"); + + //* When + let result = validate_table_references(&table, &stmt, &config, &dep_graph); + + //* Then + assert!(result.is_ok(), "eth_call should be accepted for evm-rpc"); + } + + #[test] + fn validate_table_references_rejects_eth_call_for_non_evm_dependency() { + //* Given + let config = minimal_config(); + let manifest = make_dependency_manifest("firehose", &["blocks"], &[]); + let dep_graph = make_dependency_graph("dep", manifest); + let stmt = parse_statement("SELECT dep.eth_call(NULL, NULL, NULL, 'latest') AS result"); + let table: TableName = "my_table".parse().expect("valid table name"); + + //* When + let result = validate_table_references(&table, &stmt, &config, &dep_graph); + + //* Then + let err = result.expect_err("should reject eth_call for non-evm dependency"); + assert!(matches!(err, ValidationError::EthCallNotAvailable { .. })); + } + } + + mod plan_validation { + use super::*; + + #[tokio::test] + async fn validate_incremental_constraints_rejects_underscore_alias() { + //* Given + let schema_ctx = SchemaContext::new(); + let stmt = parse_statement("SELECT 1 AS _bad"); + let table: TableName = "model".parse().expect("valid table name"); + let select_info = + crate::query::validate_select(&stmt, table.clone()).expect("valid select"); + + //* When + let result = + validate_incremental_constraints(&select_info.query, &schema_ctx, &table).await; + + //* Then + let err = result.expect_err("should reject underscore-prefixed alias"); + assert!(matches!(err, ValidationError::SqlPlanning { .. })); + } + + #[tokio::test] + async fn validate_incremental_constraints_accepts_union_all() { + //* Given + let schema_ctx = SchemaContext::new(); + // UNION ALL is allowed; plain UNION implies DISTINCT which is not incremental + let stmt = parse_statement("SELECT 1 AS id UNION ALL SELECT 2 AS id"); + let table: TableName = "model".parse().expect("valid table name"); + let select_info = + crate::query::validate_select(&stmt, table.clone()).expect("valid select"); + + //* When + let result = + validate_incremental_constraints(&select_info.query, &schema_ctx, &table).await; + + //* Then + assert!( + result.is_ok(), + "union all should be allowed: {:?}", + result.err() + ); + } + + #[tokio::test] + async fn validate_incremental_constraints_rejects_union_distinct() { + //* Given + let schema_ctx = SchemaContext::new(); + // Plain UNION implies DISTINCT which is not incremental + let stmt = parse_statement("SELECT 1 AS id UNION SELECT 2 AS id"); + let table: TableName = "model".parse().expect("valid table name"); + let select_info = + crate::query::validate_select(&stmt, table.clone()).expect("valid select"); + + //* When + let result = + validate_incremental_constraints(&select_info.query, &schema_ctx, &table).await; + + //* Then + let err = result.expect_err("union (distinct) should be rejected"); + assert!(matches!(err, ValidationError::NonIncremental { .. })); + } + } +} diff --git a/crates/core/datasets-common/src/manifest.rs b/crates/core/datasets-common/src/manifest.rs index 6fd3b8940..8424dcbaf 100644 --- a/crates/core/datasets-common/src/manifest.rs +++ b/crates/core/datasets-common/src/manifest.rs @@ -159,6 +159,23 @@ impl From for SchemaRef { } } +/// Convert Arrow `SchemaRef` to serializable `ArrowSchema`. +impl From<&SchemaRef> for ArrowSchema { + fn from(schema: &SchemaRef) -> Self { + Self { + fields: schema + .fields() + .iter() + .map(|f| Field { + name: f.name().clone(), + type_: DataType(f.data_type().clone()), + nullable: f.is_nullable(), + }) + .collect(), + } + } +} + /// Arrow field definition with name, type, and nullability. /// /// Represents a single column in a table schema. diff --git a/crates/core/monitoring/src/logging.rs b/crates/core/monitoring/src/logging.rs index 852da8100..30cd7e49e 100644 --- a/crates/core/monitoring/src/logging.rs +++ b/crates/core/monitoring/src/logging.rs @@ -81,6 +81,7 @@ const AMP_CRATES: &[&str] = &[ "auth_http", "common", "controller", + "dataset_authoring", "datasets_common", "datasets_derived", "datasets_raw", diff --git a/docs/features/dataset-authoring.md b/docs/features/dataset-authoring.md new file mode 100644 index 000000000..13824a8ba --- /dev/null +++ b/docs/features/dataset-authoring.md @@ -0,0 +1,324 @@ +--- +name: "dataset-authoring" +description: "Build and package derived datasets from amp.yaml/amp.yml configuration with SQL/Jinja templating, dependency resolution, and schema inference. Load when working with dataset build, package, or authoring workflow" +type: "feature" +status: "unstable" +components: "crate:dataset-authoring,app:ampctl" +--- + +# Dataset Authoring + +## Summary + +Dataset Authoring provides a filesystem-first workflow for defining derived datasets using YAML configuration and SQL files. The authoring pipeline parses `amp.yaml` or `amp.yml` (including the required `amp` contract version), discovers table SQL files from the `tables/` directory (default when omitted), renders Jinja templates, validates SELECT-only statements, infers Arrow schemas via DataFusion, and produces deterministic packages for deployment. The workflow integrates with the existing registry and admin API for dependency resolution and manifest registration. + +## Table of Contents + +1. [Key Concepts](#key-concepts) +2. [Usage](#usage) +3. [Configuration](#configuration) +4. [Limitations](#limitations) +5. [References](#references) + +## Key Concepts + +- **amp.yaml / amp.yml**: The YAML configuration file that defines a dataset's metadata, dependencies, tables, functions, and variables. It is the entrypoint for the authoring workflow and must include an `amp` contract version. + +- **Tables**: SQL files in the `tables/` directory (default), each containing a single SELECT query. The table name is derived from the filename (e.g., `tables/transfers.sql` creates the `transfers` table). + +- **Jinja templating**: SQL files support Jinja templates rendered at build time. Available helpers include `ref()`, `source()`, `var()`, `env_var()`, and `this`. No database access is permitted during rendering. + +- **Lockfile (amp.lock)**: Records the full transitive dependency graph with manifest hashes. Ensures reproducible builds by pinning exact versions. + +- **Package (dataset.tgz)**: A deterministic archive containing the canonical manifest, rendered SQL files, inferred schemas, and function sources. Used for distribution and deployment. + +- **Legacy bridge**: Converts the authoring manifest format to the runtime manifest format expected by existing Amp servers, enabling incremental adoption. + +## Usage + +### Package Layout + +A dataset project has this structure (use `amp.yaml` or `amp.yml`, not both): + +``` +my_dataset/ + amp.yaml # Required: dataset configuration (or amp.yml) + tables/ + transfers.sql # One SQL file per table + balances.sql + functions/ # Optional: JavaScript UDFs + my_func.js + amp.lock # Generated: dependency lockfile +``` + +### Build a Dataset + +Build a dataset from the current directory: + +```bash +ampctl dataset build --output build/ +``` + +Build from a specific directory with custom output: + +```bash +ampctl dataset build --dir my_dataset/ --output dist/ +``` + +Override Jinja variables at build time: + +```bash +ampctl dataset build --output build/ --var network=mainnet --var chain_id=1 +``` + +Build with locked dependencies (fail if `amp.lock` is missing or stale): + +```bash +ampctl dataset build --output build/ --locked +``` + +Build with offline cache only (requires amp.lock when version refs are present): + +```bash +ampctl dataset build --output build/ --offline +``` + +Build with frozen dependencies (equivalent to `--locked` + `--offline`): + +```bash +ampctl dataset build --output build/ --frozen +``` + +### Validate a Dataset + +Validate authoring inputs without writing build artifacts: + +```bash +ampctl dataset check +``` + +Validate a specific directory: + +```bash +ampctl dataset check --dir my_dataset/ +``` + +Validate with offline cache only: + +```bash +ampctl dataset check --offline +``` + +Emit structured JSON output: + +```bash +ampctl dataset check --json +``` + +### Package a Dataset + +Create a distributable archive from build output: + +```bash +ampctl dataset package +``` + +Specify custom input directory and output filename: + +```bash +ampctl dataset package --dir dist/ --output my_dataset.tgz +``` + +### Register a Dataset + +Register using the legacy bridge (from package directory): + +```bash +ampctl dataset register my_namespace/my_dataset --package build +``` + +Register with a version tag: + +```bash +ampctl dataset register my_namespace/my_dataset --package build --tag 1.0.0 +``` + +### Full Workflow Example + +```bash +# Build the dataset +ampctl dataset build --dir my_dataset/ --output build/ + +# Package for distribution +ampctl dataset package --dir build --output my_dataset.tgz + +# Register with the admin API +ampctl dataset register my_namespace/my_dataset --package build --tag 1.0.0 +``` + +### Writing SQL Files + +Each SQL file must contain exactly one SELECT statement: + +```sql +-- tables/transfers.sql +SELECT + block_num, + tx_hash, + arrow_cast(log_index, 'Int32') AS log_index, + "from", + "to", + amount +FROM {{ ref('eth', 'logs') }} +WHERE topic0 = '0xddf252ad...' +``` + +### Table Validation Constraints + +Authoring validation matches legacy `manifest.json` rules. For a table to be valid: + +- The SQL file must contain exactly one SELECT statement (no DDL/DML). +- Table references must be dependency-qualified `alias.table` and exist in the dependency manifest. Unqualified (`table`) or catalog-qualified (`catalog.schema.table`) references are rejected. +- Function references must be either built-in or qualified: `alias.func` for dependency UDFs or `self.func` for local UDFs. `eth_call` is only available for `evm-rpc` dependencies. +- Incremental constraints apply: no aggregates, `DISTINCT`, window functions, `ORDER BY`, `LIMIT/OFFSET`, outer/anti joins, or recursive queries. Inner/semi joins and `UNION ALL` are allowed (plain `UNION` implies `DISTINCT`). +- Aliases cannot start with `_` (for example, `AS _tmp`). + +### Jinja Helpers + +Available helpers in SQL templates: + +| Helper | Description | +|--------|-------------| +| `ref(alias, table)` | Reference a dependency table, returns `.
` | +| `source(alias, table)` | Alias for `ref()` (dbt compatibility) | +| `var(name, default=None)` | Get build-time variable (CLI > config > default) | +| `env_var(name, default=None)` | Get environment variable (env > default) | +| `this` | Current table name being processed | + +Variable precedence: +- `var()`: CLI `--var` > config `vars` > function default (error if missing without default) +- `env_var()`: environment > function default (error if missing without default) + +CLI `--var` keys must be declared in config `vars`; unknown keys fail build/check. + +## Configuration + +### amp.yaml / amp.yml Schema + +`amp` is the authoring contract version (semver) and is separate from the dataset `version`. + +| Field | Required | Description | +|-------|----------|-------------| +| `amp` | Yes | Authoring contract version (semver string, e.g., `1.0.0`) | +| `namespace` | Yes | Dataset namespace (lowercase, alphanumeric, underscores) | +| `name` | Yes | Dataset name (lowercase, alphanumeric, underscores) | +| `version` | Yes | Semantic version (e.g., `1.0.0`) | +| `dependencies` | No | Map of alias to `namespace/name@version` or `@hash` | +| `tables` | No | Path to directory containing table SQL files (defaults to `tables`) | +| `functions` | No | Map of function name to function definition | +| `vars` | No | Map of variable name to default value | + +### Example amp.yaml + +```yaml +amp: 1.0.0 +namespace: my_namespace +name: my_dataset +version: 1.0.0 +dependencies: + eth: my_namespace/eth_mainnet@1.0.0 + other: other_ns/other@sha256:b94d27b9... + # tables: tables # optional; defaults to "tables" +functions: + addSuffix: + input_types: [Utf8] + output_type: Utf8 + source: functions/add_suffix.js +vars: + network: mainnet +``` + +### Function Definition + +| Field | Required | Description | +|-------|----------|-------------| +| `input_types` | Yes | Array of Arrow type names (e.g., `[Int64, Utf8]`) | +| `output_type` | Yes | Arrow type name for return value | +| `source` | Yes | Path to JavaScript source file | + +### Build Output Structure + +The build command produces: + +``` +/ + manifest.json # Canonical manifest with file references + tables/ +
.sql # Rendered SQL (post-Jinja) +
.ipc # Inferred Arrow schema (IPC file format) + functions/ + .js # Copied function sources +``` + +### CLI Options + +All dataset authoring commands honor the global `--json` flag for structured output. + +#### `ampctl dataset build` + +| Flag | Default | Description | +|------|---------|-------------| +| `--dir`, `-d` | `.` | Directory containing amp.yaml or amp.yml | +| `--output`, `-o` | (required) | Output directory for build artifacts | +| `--var`, `-V` | - | Variable override (NAME=VALUE), repeatable | +| `--locked` | false | Fail if amp.lock missing or stale | +| `--offline` | false | Disable network fetches; cache-only resolution | +| `--frozen` | false | Equivalent to `--locked` + `--offline` | +| `--admin-url` | `http://localhost:1610` | Admin API URL for dependency resolution | +| `--json` | false | Emit structured JSON output | + +Notes: +- `--offline` requires `amp.lock` when version refs are present; cache misses are errors. + +#### `ampctl dataset check` + +| Flag | Default | Description | +|------|---------|-------------| +| `--dir`, `-d` | `.` | Directory containing amp.yaml or amp.yml | +| `--var`, `-V` | - | Variable override (NAME=VALUE), repeatable | +| `--locked` | false | Fail if amp.lock missing or stale | +| `--offline` | false | Disable network fetches; cache-only resolution | +| `--frozen` | false | Equivalent to `--locked` + `--offline` | +| `--admin-url` | `http://localhost:1610` | Admin API URL for dependency resolution | +| `--json` | false | Emit structured JSON output | + +#### `ampctl dataset package` + +| Flag | Default | Description | +|------|---------|-------------| +| `--dir`, `-d` | CWD (if `manifest.json` exists) | Directory containing build artifacts (required if no manifest in CWD) | +| `--output`, `-o` | `dataset.tgz` | Output archive path | +| `--json` | false | Emit structured JSON output | + +#### `ampctl dataset register` + +| Argument | Required | Description | +|----------|----------|-------------| +| `FQN` | Yes | Fully qualified name (`namespace/name`) | +| `FILE` | No | Path or URL to manifest file | +| `--package`, `-p` | No | Directory with built package (converts via bridge) | +| `--tag`, `-t` | No | Semantic version tag (e.g., `1.0.0`) | + +## Limitations + +- Dependencies must be explicitly versioned or hashed; symbolic references like `latest` or `dev` are not allowed. +- Dependency-free datasets cannot infer network configuration; include at least one dependency. +- Table SQL must satisfy the validation constraints listed above. +- Jinja is deterministic-only: no database access or filesystem/network operations during rendering. +- Using `env_var()` makes builds environment-dependent; avoid for reproducible builds. +- The legacy bridge is required until servers support the new package format directly. + +## References + +- [dataset-registry](dataset-registry.md) - Dependency: Manifest storage and version tags +- [data](data.md) - Related: Data layer overview diff --git a/playground/README.md b/playground/README.md new file mode 100644 index 000000000..bf43b32da --- /dev/null +++ b/playground/README.md @@ -0,0 +1,62 @@ +# Dataset Authoring Playground (dbt-style) + +This playground is a small dataset for trying the new authoring workflow with +table discovery and Jinja templating. + +## What it includes + +- `amp.yaml` with basic metadata, dependencies, and default `vars` +- A `tables/` directory with SELECT-only table definitions +- Jinja usage via `ref()` and `var()` + +## Prerequisites (run Amp + register raw dataset) + +1. Run Amp locally (single-node mode): + +```bash +cargo run -p ampd -- solo +``` + +2. Generate, register, and deploy the raw dataset referenced by `amp.yaml`: + +```bash +mkdir -p manifests +cargo run -p ampctl manifest generate --network mainnet --kind evm-rpc -o ./manifests/eth_mainnet.json +cargo run -p ampctl dataset register my_org/eth_mainnet ./manifests/eth_mainnet.json --tag 1.0.0 +cargo run -p ampctl dataset deploy my_org/eth_mainnet@1.0.0 +``` + +Note: you must also configure a provider for the selected network/kind +(see `docs/config.md` under **Providers**). + +## Usage + +Build the dataset (writes to a dedicated output directory): + +```bash +cargo run -p ampctl dataset build \ + --dir playground \ + --output playground/build +``` + +Override variables at build time: + +```bash +cargo run -p ampctl dataset build \ + --dir playground \ + --output playground/build \ + --var min=5 +``` + +Package the build output: + +```bash +cargo run -p ampctl dataset package --dir playground/build +``` + +## Notes + +- Every `tables/**/*.sql` file becomes a table; the table name is the file stem. +- SQL must be a single SELECT statement. +- Dependencies are resolved to their hash. Update `dependencies` in `amp.yaml` + to point at datasets available in your registry (or local amp server) diff --git a/playground/amp.lock b/playground/amp.lock new file mode 100644 index 000000000..f2364764c --- /dev/null +++ b/playground/amp.lock @@ -0,0 +1,11 @@ +version: 1 +root: + namespace: playground + name: demo + version: 0.0.0 +dependencies: + eth: 4d31610e5cad62ee287373d590cf80b9e0a63cd48ac96fe7d571d79d8759c7f8 +nodes: + 4d31610e5cad62ee287373d590cf80b9e0a63cd48ac96fe7d571d79d8759c7f8: + namespace: my_org + name: eth_mainnet diff --git a/playground/amp.yaml b/playground/amp.yaml new file mode 100644 index 000000000..b5972e0f5 --- /dev/null +++ b/playground/amp.yaml @@ -0,0 +1,8 @@ +amp: 1.0.0 +namespace: playground +name: demo +version: 0.0.0 +dependencies: + eth: my_org/eth_mainnet@1.0.0 +vars: + min: "1" diff --git a/playground/tables/sample.sql b/playground/tables/sample.sql new file mode 100644 index 000000000..440430eac --- /dev/null +++ b/playground/tables/sample.sql @@ -0,0 +1,7 @@ +SELECT + t.block_num, + t.tx_hash, + t.from, + t.value +FROM {{ ref("eth", "transactions") }} AS t +WHERE t.value >= CAST({{ var("min", "1") }} AS BIGINT) diff --git a/tests/Cargo.toml b/tests/Cargo.toml index c05382add..14b394c0a 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -12,6 +12,8 @@ name = "tests" [dependencies] alloy.workspace = true +async-trait.workspace = true +datafusion.workspace = true amp-client = { path = "../crates/clients/flight" } amp-data-store = { path = "../crates/core/data-store" } amp-object-store = { path = "../crates/core/object-store" } @@ -29,6 +31,7 @@ ctor = "0.5.0" amp-dataset-store = { path = "../crates/core/dataset-store" } amp-datasets-registry = { path = "../crates/core/datasets-registry" } amp-providers-registry = { path = "../crates/core/providers-registry" } +dataset-authoring = { path = "../crates/core/dataset-authoring" } datasets-common = { path = "../crates/core/datasets-common" } datasets-derived = { path = "../crates/core/datasets-derived" } dotenvy.workspace = true diff --git a/tests/src/tests/it_dataset_authoring.rs b/tests/src/tests/it_dataset_authoring.rs new file mode 100644 index 000000000..6de5e5a2a --- /dev/null +++ b/tests/src/tests/it_dataset_authoring.rs @@ -0,0 +1,1576 @@ +//! Integration tests for dataset authoring workflow. +//! +//! These tests validate the complete authoring pipeline: +//! - Building datasets from amp.yaml and SQL templates +//! - Packaging datasets into deterministic archives +//! - Converting to legacy format via the bridge +//! +//! Note: These tests use the `dataset-authoring` crate directly rather than +//! the CLI to avoid the need for a running admin API for dependency resolution. +//! For tests that require dependency resolution via admin API, see the +//! `it_admin_api_datasets_register.rs` tests. + +use std::{collections::BTreeMap, fs, path::Path}; + +use dataset_authoring::{ + arrow_ipc, arrow_json, + bridge::LegacyBridge, + canonical, + config::AmpYaml, + discovery::discover_tables, + files::{FileRef, hash_file, normalize_path}, + jinja::{JinjaContext, render_sql}, + lockfile::{Lockfile, RootInfo}, + manifest::{AuthoringManifest, TableDef}, + package::PackageBuilder, + query::validate_select, + resolver::DependencyGraph, + schema::SchemaContext, +}; +use datasets_common::hash::Hash; +use datasets_derived::sql_str::SqlStr; +use monitoring::logging; +use tempfile::TempDir; + +// ============================================================ +// Test fixtures and helpers +// ============================================================ + +/// Create a minimal amp.yaml for testing without dependencies. +fn create_minimal_amp_yaml(namespace: &str, name: &str) -> String { + indoc::formatdoc! {r#" + amp: "1.0.0" + namespace: {namespace} + name: {name} + version: "1.0.0" + tables: tables + "#} +} + +/// Create a simple SELECT SQL file for testing (dbt-style model). +fn create_simple_sql() -> String { + "SELECT 1 AS id, 'test' AS name".to_string() +} + +/// Set up a minimal dataset project directory. +fn setup_minimal_project(dir: &Path, namespace: &str, name: &str) { + // Create directories + let tables_dir = dir.join("tables"); + fs::create_dir_all(&tables_dir).expect("should create tables dir"); + + // Write amp.yaml + let amp_yaml = create_minimal_amp_yaml(namespace, name); + fs::write(dir.join("amp.yaml"), amp_yaml).expect("should write amp.yaml"); + + // Write SQL file (SELECT-only, table name from filename) + let sql = create_simple_sql(); + fs::write(tables_dir.join("transfers.sql"), sql).expect("should write sql file"); +} + +/// Set up a dataset project with variables for Jinja testing. +fn setup_project_with_vars(dir: &Path) { + let tables_dir = dir.join("tables"); + fs::create_dir_all(&tables_dir).expect("should create tables dir"); + + // amp.yaml with variables + let amp_yaml = indoc::indoc! {r#" + amp: "1.0.0" + namespace: test_ns + name: test_dataset + version: "1.0.0" + tables: tables + vars: + filter_value: "100" + "#}; + fs::write(dir.join("amp.yaml"), amp_yaml).expect("should write amp.yaml"); + + // SQL with Jinja variable (SELECT-only, dbt-style) + let sql = indoc::indoc! {r#" + SELECT id, amount + FROM (SELECT 1 AS id, 200 AS amount) + WHERE amount > {{ var('filter_value') }} + "#}; + fs::write(tables_dir.join("filtered.sql"), sql).expect("should write sql file"); +} + +// ============================================================ +// Configuration parsing tests +// ============================================================ + +#[test] +fn parse_amp_yaml_from_file() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + setup_minimal_project(dir.path(), "test_ns", "test_dataset"); + + //* When + let content = fs::read_to_string(dir.path().join("amp.yaml")).expect("should read amp.yaml"); + let config = AmpYaml::from_yaml(&content); + + //* Then + let config = config.expect("should parse amp.yaml"); + assert_eq!(config.namespace.to_string(), "test_ns"); + assert_eq!(config.name.to_string(), "test_dataset"); + assert_eq!(config.version.to_string(), "1.0.0"); + assert_eq!(config.tables.to_string_lossy(), "tables"); + + // Discover tables from the directory + let discovered = discover_tables(dir.path(), &config.tables).expect("should discover tables"); + assert_eq!(discovered.len(), 1); + assert!(discovered.contains_key(&"transfers".parse().unwrap())); +} + +// ============================================================ +// Jinja rendering tests +// ============================================================ + +#[test] +fn jinja_renders_sql_with_variables() { + logging::init(); + + //* Given + let dir = TempDir::new().expect("should create temp dir"); + setup_project_with_vars(dir.path()); + + let sql_template = + fs::read_to_string(dir.path().join("tables/filtered.sql")).expect("should read sql"); + + let yaml_vars: std::collections::HashMap = + [("filter_value".to_string(), "100".to_string())] + .into_iter() + .collect(); + + let ctx = JinjaContext::builder().with_yaml_vars(yaml_vars).build(); + + //* When + let rendered = render_sql(&sql_template, "filtered", &ctx); + + //* Then + let rendered = rendered.expect("should render sql"); + assert!( + rendered.contains("WHERE amount > 100"), + "should substitute variable, got: {}", + rendered + ); +} + +#[test] +fn jinja_cli_vars_override_yaml_vars() { + logging::init(); + + //* Given + let template = "SELECT * FROM source WHERE x > {{ var('limit') }}"; + + let yaml_vars: std::collections::HashMap = + [("limit".to_string(), "10".to_string())] + .into_iter() + .collect(); + let cli_vars = vec![("limit".to_string(), "50".to_string())]; + + let ctx = JinjaContext::builder() + .with_yaml_vars(yaml_vars) + .with_cli_vars(cli_vars) + .build(); + + //* When + let rendered = render_sql(template, "test", &ctx); + + //* Then + let rendered = rendered.expect("should render sql"); + assert!( + rendered.contains("WHERE x > 50"), + "CLI vars should override YAML vars, got: {}", + rendered + ); +} + +// ============================================================ +// SELECT validation tests (dbt-style models) +// ============================================================ + +#[test] +fn validate_select_accepts_valid_statement() { + logging::init(); + + //* Given + let sql: SqlStr = "SELECT id, amount FROM source".parse().expect("valid sql"); + let stmt = common::sql::parse(&sql).expect("should parse"); + + //* When + let result = validate_select(&stmt, "transfers".parse().expect("valid table name")); + + //* Then + let select_info = result.expect("should validate SELECT"); + assert_eq!(select_info.table_name.to_string(), "transfers"); +} + +#[test] +fn validate_select_rejects_ctas() { + logging::init(); + + //* Given + let sql: SqlStr = "CREATE TABLE transfers AS SELECT id FROM source" + .parse() + .expect("valid sql"); + let stmt = common::sql::parse(&sql).expect("should parse"); + + //* When + let result = validate_select(&stmt, "transfers".parse().expect("valid table name")); + + //* Then + assert!(result.is_err(), "should reject CTAS statement"); +} + +#[test] +fn validate_select_rejects_insert() { + logging::init(); + + //* Given + let sql: SqlStr = "INSERT INTO transfers VALUES (1, 'test')" + .parse() + .expect("valid sql"); + let stmt = common::sql::parse(&sql).expect("should parse"); + + //* When + let result = validate_select(&stmt, "transfers".parse().expect("valid table name")); + + //* Then + assert!(result.is_err(), "should reject INSERT statement"); +} + +// ============================================================ +// Schema inference tests +// ============================================================ + +#[tokio::test] +async fn infer_schema_from_select_query() { + logging::init(); + + //* Given + let schema_ctx = SchemaContext::new(); + + // Parse a SELECT statement and extract the query + let sql: SqlStr = "SELECT 1 AS id, 'test' AS name, true AS active" + .parse() + .expect("valid sql"); + let stmt = common::sql::parse(&sql).expect("should parse"); + let select_info = + validate_select(&stmt, "test".parse().expect("valid table name")).expect("should validate"); + + //* When + let result = schema_ctx.infer_schema(&select_info.query).await; + + //* Then + let schema = result.expect("should infer schema"); + assert_eq!(schema.fields().len(), 3); + + // Check field names + let field_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert!(field_names.contains(&"id")); + assert!(field_names.contains(&"name")); + assert!(field_names.contains(&"active")); +} + +// ============================================================ +// File utilities tests +// ============================================================ + +#[test] +fn hash_file_produces_consistent_hash() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let file_path = dir.path().join("test.txt"); + fs::write(&file_path, "hello world").expect("should write file"); + + //* When + let hash1 = hash_file(&file_path).expect("should hash file"); + let hash2 = hash_file(&file_path).expect("should hash file again"); + + //* Then + assert_eq!(hash1, hash2, "same content should produce same hash"); +} + +#[test] +fn normalize_path_produces_posix_path() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let base = dir.path(); + let file_path = base.join("sql").join("test.sql"); + fs::create_dir_all(base.join("sql")).expect("should create dir"); + fs::write(&file_path, "test").expect("should write file"); + + //* When + let normalized = normalize_path(&file_path, base); + + //* Then + let normalized = normalized.expect("should normalize path"); + assert_eq!(normalized, "sql/test.sql"); + assert!(!normalized.contains("\\"), "should use POSIX separators"); +} + +// ============================================================ +// Canonical JSON tests +// ============================================================ + +#[test] +fn canonical_json_is_deterministic() { + //* Given + let value1: serde_json::Value = serde_json::json!({ + "z": 1, + "a": 2, + "m": {"b": 3, "a": 4} + }); + let value2: serde_json::Value = serde_json::json!({ + "a": 2, + "m": {"a": 4, "b": 3}, + "z": 1 + }); + + //* When + let canonical1 = canonical::canonicalize(&value1).expect("should canonicalize"); + let canonical2 = canonical::canonicalize(&value2).expect("should canonicalize"); + + //* Then + assert_eq!( + canonical1, canonical2, + "equivalent JSON should canonicalize to same string" + ); + + // Verify keys are sorted + assert!( + canonical1.find("\"a\"").unwrap() < canonical1.find("\"m\"").unwrap(), + "keys should be sorted alphabetically" + ); +} + +// ============================================================ +// Lockfile tests +// ============================================================ + +#[test] +fn lockfile_roundtrip_preserves_content() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let lockfile_path = dir.path().join("amp.lock"); + + let root = RootInfo { + namespace: "test_ns".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + version: "1.0.0".parse().expect("valid version"), + }; + + // Create empty dependency graph for testing + let dep_graph = DependencyGraph::default(); + let lockfile = Lockfile::from_graph(root, &dep_graph); + + //* When + lockfile + .write(&lockfile_path) + .expect("should write lockfile"); + let loaded = Lockfile::read(&lockfile_path).expect("should read lockfile"); + + //* Then + assert_eq!(lockfile, loaded, "lockfile should roundtrip correctly"); +} + +#[test] +fn lockfile_is_deterministic() { + //* Given + let root = RootInfo { + namespace: "test_ns".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + version: "1.0.0".parse().expect("valid version"), + }; + + let dep_graph = DependencyGraph::default(); + + //* When + let lockfile1 = Lockfile::from_graph(root.clone(), &dep_graph); + let lockfile2 = Lockfile::from_graph(root, &dep_graph); + + let yaml1 = serde_yaml::to_string(&lockfile1).expect("should serialize"); + let yaml2 = serde_yaml::to_string(&lockfile2).expect("should serialize"); + + //* Then + assert_eq!( + yaml1, yaml2, + "lockfile serialization should be deterministic" + ); +} + +// ============================================================ +// Package assembly tests +// ============================================================ + +#[test] +fn package_builder_creates_deterministic_archive() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + setup_build_output(dir.path()); + + //* When + let builder1 = PackageBuilder::from_directory(dir.path()).expect("should create builder"); + let builder2 = PackageBuilder::from_directory(dir.path()).expect("should create builder again"); + + let hash1 = builder1.hash().expect("should compute hash"); + let hash2 = builder2.hash().expect("should compute hash again"); + + //* Then + assert_eq!( + hash1, hash2, + "same content should produce same package hash" + ); +} + +#[test] +fn package_builder_includes_all_required_files() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + setup_build_output(dir.path()); + + //* When + let builder = PackageBuilder::from_directory(dir.path()).expect("should create builder"); + let entries = builder.entries(); + + //* Then + let paths: Vec<&str> = entries.iter().map(|e| e.path.as_str()).collect(); + assert!(paths.contains(&"manifest.json"), "should include manifest"); + assert!( + paths.iter().any(|p| p.starts_with("tables/")), + "should include table files" + ); +} + +#[test] +fn package_writes_valid_tgz() { + //* Given + let dir = TempDir::new().expect("should create temp dir"); + setup_build_output(dir.path()); + + let output_path = dir.path().join("output.tgz"); + let builder = PackageBuilder::from_directory(dir.path()).expect("should create builder"); + + //* When + builder.write_to(&output_path).expect("should write tgz"); + + //* Then + assert!(output_path.exists(), "tgz file should exist"); + assert!( + fs::metadata(&output_path) + .expect("should get metadata") + .len() + > 0, + "tgz file should have content" + ); +} + +/// Helper to set up a build output directory for packaging tests. +fn setup_build_output(dir: &Path) { + use std::sync::Arc; + + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + + let tables_dir = dir.join("tables"); + fs::create_dir_all(&tables_dir).expect("should create tables dir"); + + // Create manifest.json + let manifest = serde_json::json!({ + "namespace": "test_ns", + "name": "test_dataset", + "kind": "derived", + "dependencies": {}, + "tables": { + "transfers": { + "sql": {"path": "tables/transfers.sql", "hash": "sha256:abc123"}, + "ipc": {"path": "tables/transfers.ipc", "hash": "sha256:def456"}, + "network": "mainnet" + } + }, + "functions": {} + }); + fs::write( + dir.join("manifest.json"), + serde_json::to_string_pretty(&manifest).unwrap(), + ) + .expect("should write manifest"); + + // Create SQL file + fs::write( + tables_dir.join("transfers.sql"), + "CREATE TABLE transfers AS SELECT 1 AS id", + ) + .expect("should write sql"); + + // Create IPC schema file using Arrow IPC format + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + arrow_ipc::write_ipc_schema(&schema, tables_dir.join("transfers.ipc")) + .expect("should write ipc schema"); +} + +// ============================================================ +// Legacy bridge tests +// ============================================================ + +#[test] +fn legacy_bridge_converts_manifest_with_inline_content() { + use std::sync::Arc; + + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let tables_dir = dir.path().join("tables"); + fs::create_dir_all(&tables_dir).expect("should create tables dir"); + + // Create SQL file + fs::write( + tables_dir.join("transfers.sql"), + "CREATE TABLE transfers AS SELECT id, amount FROM source.transactions", + ) + .expect("should write sql"); + + // Create IPC schema file using Arrow IPC format + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt64, false), + Field::new("amount", DataType::UInt64, false), + ])); + arrow_ipc::write_ipc_schema(&schema, tables_dir.join("transfers.ipc")) + .expect("should write ipc schema"); + + let test_hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + let mut manifest = AuthoringManifest { + namespace: "test_ns".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + kind: "derived".to_string(), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + }; + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + test_hash.clone(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), test_hash), + network: "mainnet".parse().expect("valid network"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.convert(&manifest); + + //* Then + let legacy = result.expect("conversion should succeed"); + assert_eq!(legacy.tables.len(), 1); + + let table = legacy + .tables + .get(&"transfers".parse().unwrap()) + .expect("should have table"); + + // Verify SQL is inlined + match &table.input { + datasets_derived::manifest::TableInput::View(view) => { + assert!( + view.sql.as_str().contains("CREATE TABLE transfers"), + "SQL should be inlined" + ); + } + } + + // Verify schema is loaded + assert_eq!(table.schema.arrow.fields.len(), 2); +} + +#[test] +fn legacy_bridge_produces_valid_json() { + use std::sync::Arc; + + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + + //* Given + let dir = TempDir::new().expect("should create temp dir"); + let tables_dir = dir.path().join("tables"); + fs::create_dir_all(&tables_dir).expect("should create tables dir"); + + fs::write( + tables_dir.join("transfers.sql"), + "CREATE TABLE transfers AS SELECT 1", + ) + .expect("should write sql"); + + // Create IPC schema file using Arrow IPC format + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + arrow_ipc::write_ipc_schema(&schema, tables_dir.join("transfers.ipc")) + .expect("should write ipc schema"); + + let test_hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + let mut manifest = AuthoringManifest { + namespace: "test_ns".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + kind: "derived".to_string(), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + }; + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + test_hash.clone(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), test_hash), + network: "mainnet".parse().expect("valid network"), + }, + ); + + let bridge = LegacyBridge::new(dir.path()); + + //* When + let result = bridge.to_json(&manifest); + + //* Then + let json_str = result.expect("to_json should succeed"); + let parsed: serde_json::Value = serde_json::from_str(&json_str).expect("should be valid JSON"); + + // Verify structure matches derived.spec.json requirements + assert_eq!(parsed["kind"], "manifest"); + assert!(parsed["tables"].is_object()); + assert!(parsed["tables"]["transfers"]["input"]["sql"].is_string()); + assert!(parsed["tables"]["transfers"]["schema"]["arrow"]["fields"].is_array()); + assert!(parsed["tables"]["transfers"]["network"].is_string()); +} + +// ============================================================ +// Full pipeline integration tests +// ============================================================ + +#[tokio::test] +async fn full_build_pipeline_without_dependencies() { + logging::init(); + + //* Given + let project_dir = TempDir::new().expect("should create project dir"); + let output_dir = TempDir::new().expect("should create output dir"); + + setup_minimal_project(project_dir.path(), "integration", "test_build"); + + // Parse configuration + let config_content = + fs::read_to_string(project_dir.path().join("amp.yaml")).expect("should read amp.yaml"); + let config = AmpYaml::from_yaml(&config_content).expect("should parse amp.yaml"); + + // Discover tables + let discovered_tables = + discover_tables(project_dir.path(), &config.tables).expect("should discover tables"); + + // Set up output directories + let sql_output_dir = output_dir.path().join("sql"); + fs::create_dir_all(&sql_output_dir).expect("should create sql output dir"); + + // Build Jinja context + let jinja_ctx = JinjaContext::builder().build(); + + // Build schema context (empty, no dependencies) + let schema_ctx = SchemaContext::new(); + + //* When + // Process each discovered table + for (table_name, sql_path) in &discovered_tables { + // Read template + let template_path = project_dir.path().join(sql_path); + let template = fs::read_to_string(&template_path).expect("should read sql template"); + + // Render Jinja + let rendered = + render_sql(&template, table_name.as_str(), &jinja_ctx).expect("should render sql"); + + // Parse SQL as a SELECT query (dbt-style) + let sql_str: SqlStr = rendered.parse().expect("valid sql"); + let stmt = common::sql::parse(&sql_str).expect("should parse sql"); + + // Extract query from SELECT statement + let query = match &stmt { + datafusion::sql::parser::Statement::Statement(inner) => match inner.as_ref() { + datafusion::sql::sqlparser::ast::Statement::Query(q) => *q.clone(), + _ => panic!("expected SELECT statement"), + }, + _ => panic!("expected SELECT statement"), + }; + + // Infer schema using the query + let schema = schema_ctx + .infer_schema(&query) + .await + .expect("should infer schema"); + + // Write output files + let sql_output_path = sql_output_dir.join(format!("{}.sql", table_name)); + fs::write(&sql_output_path, &rendered).expect("should write sql"); + + let schema_output_path = sql_output_dir.join(format!("{}.schema.json", table_name)); + let arrow_schema: datasets_common::manifest::ArrowSchema = (&schema).into(); + arrow_json::write_schema_file(&arrow_schema, &schema_output_path) + .expect("should write schema"); + } + + //* Then + // Verify output files exist + assert!( + sql_output_dir.join("transfers.sql").exists(), + "SQL file should exist" + ); + assert!( + sql_output_dir.join("transfers.schema.json").exists(), + "Schema file should exist" + ); + + // Verify schema content + let schema = arrow_json::read_schema_file(sql_output_dir.join("transfers.schema.json")) + .expect("should read schema"); + assert!(!schema.fields.is_empty(), "schema should have fields"); +} + +#[tokio::test] +async fn full_package_and_bridge_pipeline() { + logging::init(); + + //* Given + let dir = TempDir::new().expect("should create temp dir"); + + // Set up a complete build output + setup_build_output(dir.path()); + + //* When + // 1. Create package + let package_path = dir.path().join("dataset.tgz"); + let builder = + PackageBuilder::from_directory(dir.path()).expect("should create package builder"); + builder + .write_to(&package_path) + .expect("should write package"); + let package_hash = builder.hash().expect("should compute package hash"); + + // 2. Read the manifest for bridge conversion + let manifest_content = + fs::read_to_string(dir.path().join("manifest.json")).expect("should read manifest"); + let _manifest_value: serde_json::Value = + serde_json::from_str(&manifest_content).expect("should parse manifest json"); + + // Parse the authoring manifest (simplified for this test) + let test_hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + let mut authoring_manifest = AuthoringManifest { + namespace: "test_ns".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + kind: "derived".to_string(), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + }; + authoring_manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + test_hash.clone(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), test_hash), + network: "mainnet".parse().expect("valid network"), + }, + ); + + // 3. Convert to legacy format + let bridge = LegacyBridge::new(dir.path()); + let legacy_json = bridge + .to_json(&authoring_manifest) + .expect("should convert to legacy json"); + + //* Then + // Verify package was created + assert!(package_path.exists(), "package file should exist"); + assert!( + !package_hash.to_string().is_empty(), + "package hash should not be empty" + ); + + // Verify legacy JSON is valid and has expected structure + let legacy_value: serde_json::Value = + serde_json::from_str(&legacy_json).expect("legacy json should be valid"); + assert_eq!(legacy_value["kind"], "manifest"); + assert!(legacy_value["tables"]["transfers"]["input"]["sql"].is_string()); + + // Verify SQL content is inlined in legacy format + let inlined_sql = legacy_value["tables"]["transfers"]["input"]["sql"] + .as_str() + .expect("sql should be string"); + assert!( + inlined_sql.contains("CREATE TABLE transfers"), + "SQL should be inlined in legacy format" + ); + + tracing::info!( + package_hash = %package_hash, + "Pipeline test completed successfully" + ); +} + +// ============================================================ +// Dependency resolution integration tests +// ============================================================ + +// ============================================================ +// Canonical manifest tests +// ============================================================ + +#[test] +fn canonical_manifest_hash_matches_identity() { + logging::init(); + + //* Given + // Create a manifest and serialize it to canonical JSON + let test_hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + let mut manifest = AuthoringManifest { + namespace: "test_ns".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + kind: "derived".to_string(), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + }; + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + test_hash.clone(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), test_hash), + network: "mainnet".parse().expect("valid network"), + }, + ); + + //* When + let canonical_json = manifest + .canonical_json() + .expect("should produce canonical JSON"); + let identity = manifest.identity().expect("should compute identity"); + + // Compute SHA-256 of the canonical JSON bytes + let computed_hash = datasets_common::hash::hash(canonical_json.as_bytes()); + + //* Then + assert_eq!( + computed_hash, identity, + "SHA-256 of canonical JSON bytes must equal manifest identity" + ); +} + +#[test] +fn canonical_manifest_is_deterministic() { + logging::init(); + + //* Given + let dir = TempDir::new().expect("should create temp dir"); + + // Create a manifest + let test_hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + let mut manifest = AuthoringManifest { + namespace: "test_ns".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + kind: "derived".to_string(), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + }; + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + test_hash.clone(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), test_hash), + network: "mainnet".parse().expect("valid network"), + }, + ); + + // Write to files twice + let path1 = dir.path().join("manifest1.json"); + let path2 = dir.path().join("manifest2.json"); + + //* When + let json1 = manifest.canonical_json().expect("should produce JSON 1"); + let json2 = manifest.canonical_json().expect("should produce JSON 2"); + + fs::write(&path1, &json1).expect("should write manifest 1"); + fs::write(&path2, &json2).expect("should write manifest 2"); + + // Read back and compare + let bytes1 = fs::read(&path1).expect("should read manifest 1"); + let bytes2 = fs::read(&path2).expect("should read manifest 2"); + + //* Then + assert_eq!( + bytes1, bytes2, + "canonical manifest JSON must be byte-for-byte identical across invocations" + ); + + // Also verify the identities match + let identity1 = manifest.identity().expect("should compute identity 1"); + let identity2 = manifest.identity().expect("should compute identity 2"); + assert_eq!( + identity1, identity2, + "manifest identities must be identical" + ); +} + +// ============================================================ +// Function path canonicalization tests +// ============================================================ + +#[test] +fn legacy_bridge_reads_function_from_canonical_path() { + use std::sync::Arc; + + use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, Schema}; + + logging::init(); + + //* Given + // Set up a directory structure with functions at canonical paths + let dir = TempDir::new().expect("should create temp dir"); + let tables_dir = dir.path().join("tables"); + let functions_dir = dir.path().join("functions"); + fs::create_dir_all(&tables_dir).expect("should create tables dir"); + fs::create_dir_all(&functions_dir).expect("should create functions dir"); + + // Create files at canonical locations + fs::write( + tables_dir.join("transfers.sql"), + "CREATE TABLE transfers AS SELECT 1", + ) + .expect("should write sql"); + + // Create IPC schema file using Arrow IPC format + let schema = Arc::new(Schema::new(vec![Field::new( + "id", + ArrowDataType::Int64, + false, + )])); + arrow_ipc::write_ipc_schema(&schema, tables_dir.join("transfers.ipc")) + .expect("should write ipc schema"); + + // Function file at canonical path (functions/.js) + let function_content = "function decode(input) { return input.toString(); }"; + fs::write(functions_dir.join("decode.js"), function_content).expect("should write function"); + + // Create a manifest with function using canonical path + let test_hash: Hash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + .parse() + .expect("valid hash"); + + use dataset_authoring::manifest::AuthoringFunctionDef; + use datasets_common::manifest::DataType; + + // Parse DataType from string using serde (avoids needing arrow crate) + let binary_type: DataType = + serde_json::from_value(serde_json::Value::String("Binary".to_string())) + .expect("valid type"); + let utf8_type: DataType = + serde_json::from_value(serde_json::Value::String("Utf8".to_string())).expect("valid type"); + + let mut manifest = AuthoringManifest { + namespace: "test_ns".parse().expect("valid namespace"), + name: "test_dataset".parse().expect("valid name"), + kind: "derived".to_string(), + dependencies: BTreeMap::new(), + tables: BTreeMap::new(), + functions: BTreeMap::new(), + }; + manifest.tables.insert( + "transfers".parse().expect("valid table name"), + TableDef { + sql: Some(FileRef::new( + "tables/transfers.sql".to_string(), + test_hash.clone(), + )), + ipc: FileRef::new("tables/transfers.ipc".to_string(), test_hash.clone()), + network: "mainnet".parse().expect("valid network"), + }, + ); + manifest.functions.insert( + "decode".parse().expect("valid func name"), + AuthoringFunctionDef { + input_types: vec![binary_type], + output_type: utf8_type, + // The manifest uses canonical path - this is what the manifest builder produces + source: FileRef::new("functions/decode.js".to_string(), test_hash), + }, + ); + + //* When + let bridge = LegacyBridge::new(dir.path()); + let result = bridge.convert(&manifest); + + //* Then + let legacy = result.expect("conversion should succeed"); + + // Verify function was read from canonical path and inlined + let func = legacy + .functions + .get(&"decode".parse().unwrap()) + .expect("should have function"); + + assert!( + func.source.source.contains("function decode(input)"), + "function source should be inlined from canonical path" + ); + assert_eq!( + func.source.filename.as_str(), + "decode.js", + "function filename should use canonical name" + ); +} + +// ============================================================ +// Package auto-detect tests +// ============================================================ + +#[test] +fn package_auto_detects_cwd_with_manifest() { + logging::init(); + + //* Given + // Create a directory with build output (manifest.json + sql/) + let dir = TempDir::new().expect("should create temp dir"); + setup_build_output(dir.path()); + + // Save original working directory + let original_cwd = std::env::current_dir().expect("should get cwd"); + + // Change to the build output directory + std::env::set_current_dir(dir.path()).expect("should change to temp dir"); + + //* When + // Test auto-detect logic: CWD has manifest.json, so it should work + let cwd = std::env::current_dir().expect("should get cwd"); + let manifest_path = cwd.join("manifest.json"); + let dir_to_use = if manifest_path.exists() { + Some(cwd.clone()) + } else { + None + }; + + //* Then + assert!( + dir_to_use.is_some(), + "should auto-detect CWD when manifest.json exists" + ); + let detected_dir = dir_to_use.expect("should have dir"); + assert_eq!(detected_dir, cwd, "auto-detected dir should be CWD"); + + // Verify we can create a package from the auto-detected directory + let builder = PackageBuilder::from_directory(&detected_dir).expect("should create builder"); + let entries = builder.entries(); + assert!( + entries.iter().any(|e| e.path == "manifest.json"), + "should include manifest.json" + ); + + // Restore original working directory + std::env::set_current_dir(original_cwd).expect("should restore cwd"); +} + +#[test] +fn package_auto_detect_fails_without_manifest() { + logging::init(); + + //* Given + // Create an empty directory without manifest.json + let dir = TempDir::new().expect("should create temp dir"); + + // Save original working directory + let original_cwd = std::env::current_dir().expect("should get cwd"); + + // Change to the empty directory + std::env::set_current_dir(dir.path()).expect("should change to temp dir"); + + //* When + let cwd = std::env::current_dir().expect("should get cwd"); + let manifest_path = cwd.join("manifest.json"); + let has_manifest = manifest_path.exists(); + + //* Then + assert!( + !has_manifest, + "empty directory should not have manifest.json" + ); + // In the actual CLI, this would return Error::MissingDirectory + + // Restore original working directory + std::env::set_current_dir(original_cwd).expect("should restore cwd"); +} + +mod dependency_resolution { + use std::collections::BTreeMap; + + use async_trait::async_trait; + use dataset_authoring::{ + cache::Cache, + canonical::hash_canonical, + resolver::{DependencyGraph, ManifestFetcher, ResolveError, Resolver}, + }; + use datasets_common::hash::Hash; + use datasets_derived::deps::{DepAlias, DepReference}; + use monitoring::logging; + use tempfile::TempDir; + + /// Mock fetcher for integration testing. + /// + /// Stores manifests by reference string and by hash, simulating + /// an admin API without needing a real server. + struct MockFetcher { + /// Manifests keyed by reference string (e.g., "ns/name@version"). + by_reference: BTreeMap, + /// Manifests keyed by content hash. + by_hash: BTreeMap, + } + + impl MockFetcher { + fn new() -> Self { + Self { + by_reference: BTreeMap::new(), + by_hash: BTreeMap::new(), + } + } + + /// Registers a manifest for both version and hash-based lookup. + fn register(&mut self, reference: &str, manifest: serde_json::Value) { + let hash = hash_canonical(&manifest).expect("should hash manifest"); + + self.by_reference + .insert(reference.to_string(), manifest.clone()); + self.by_hash.insert(hash, manifest); + } + } + + #[async_trait] + impl ManifestFetcher for MockFetcher { + async fn fetch_by_version( + &self, + reference: &DepReference, + ) -> Result { + self.by_reference + .get(&reference.to_string()) + .cloned() + .ok_or_else(|| ResolveError::NotFound { + reference: reference.to_string(), + }) + } + + async fn fetch_by_hash( + &self, + hash: &Hash, + ) -> Result, ResolveError> { + Ok(self.by_hash.get(hash).cloned()) + } + } + + /// Creates a minimal manifest with no dependencies. + fn create_leaf_manifest(name: &str) -> serde_json::Value { + serde_json::json!({ + "kind": "manifest", + "dependencies": {}, + "tables": { + name: { + "input": { "sql": format!("CREATE TABLE {name} AS SELECT 1") }, + "schema": { "arrow": { "fields": [] } }, + "network": "mainnet" + } + }, + "functions": {} + }) + } + + /// Creates a manifest with a dependency on another manifest. + fn create_manifest_with_dep( + name: &str, + dep_alias: &str, + dep_reference: &str, + ) -> serde_json::Value { + let mut deps = serde_json::Map::new(); + deps.insert(dep_alias.to_string(), serde_json::json!(dep_reference)); + + serde_json::json!({ + "kind": "manifest", + "dependencies": deps, + "tables": { + name: { + "input": { "sql": format!("CREATE TABLE {name} AS SELECT * FROM {dep_alias}.data") }, + "schema": { "arrow": { "fields": [] } }, + "network": "mainnet" + } + }, + "functions": {} + }) + } + + #[tokio::test] + async fn resolve_single_dependency_populates_cache() { + logging::init(); + + //* Given + let temp_dir = TempDir::new().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let mut fetcher = MockFetcher::new(); + + // Register a leaf manifest + let manifest = create_leaf_manifest("transfers"); + let manifest_hash = hash_canonical(&manifest).expect("should hash"); + fetcher.register("source/dataset@1.0.0", manifest); + + let resolver = Resolver::new(fetcher, cache.clone()); + + let mut deps: BTreeMap = BTreeMap::new(); + deps.insert( + "source".parse().unwrap(), + "source/dataset@1.0.0".parse().unwrap(), + ); + + //* When + let graph: Result = resolver.resolve_all(&deps).await; + + //* Then + let graph = graph.expect("resolution should succeed"); + + // Verify graph structure + assert_eq!(graph.len(), 1, "should have one dependency"); + assert!(graph.direct.contains_key(&"source".parse().unwrap())); + + // Verify cache was populated + assert!( + cache.contains(&manifest_hash), + "cache should contain resolved manifest" + ); + } + + #[tokio::test] + async fn resolve_transitive_dependencies_populates_cache() { + logging::init(); + + //* Given + let temp_dir = TempDir::new().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let mut fetcher = MockFetcher::new(); + + // Create dependency chain: A -> B -> C + // C is a leaf (no dependencies) + let manifest_c = create_leaf_manifest("c_table"); + let hash_c = hash_canonical(&manifest_c).expect("should hash c"); + fetcher.register("ns/c@1.0.0", manifest_c); + + // B depends on C + let manifest_b = create_manifest_with_dep("b_table", "c_dep", "ns/c@1.0.0"); + let hash_b = hash_canonical(&manifest_b).expect("should hash b"); + fetcher.register("ns/b@1.0.0", manifest_b); + + // A depends on B + let manifest_a = create_manifest_with_dep("a_table", "b_dep", "ns/b@1.0.0"); + let hash_a = hash_canonical(&manifest_a).expect("should hash a"); + fetcher.register("ns/a@1.0.0", manifest_a); + + let resolver = Resolver::new(fetcher, cache.clone()); + + let mut deps: BTreeMap = BTreeMap::new(); + deps.insert("a".parse().unwrap(), "ns/a@1.0.0".parse().unwrap()); + + //* When + let graph: Result = resolver.resolve_all(&deps).await; + + //* Then + let graph = graph.expect("resolution should succeed"); + + // Verify all transitive dependencies are resolved + assert_eq!(graph.len(), 3, "should have three dependencies (A, B, C)"); + + // Verify cache contains all manifests + assert!(cache.contains(&hash_a), "cache should contain A"); + assert!(cache.contains(&hash_b), "cache should contain B"); + assert!(cache.contains(&hash_c), "cache should contain C"); + + // Verify topological order: C before B before A + let order = graph.topological_order().expect("should compute order"); + assert_eq!(order.len(), 3); + + let pos_a = order.iter().position(|h| h == &hash_a).unwrap(); + let pos_b = order.iter().position(|h| h == &hash_b).unwrap(); + let pos_c = order.iter().position(|h| h == &hash_c).unwrap(); + + assert!(pos_c < pos_b, "C should come before B"); + assert!(pos_b < pos_a, "B should come before A"); + + tracing::info!( + hash_a = %hash_a, + hash_b = %hash_b, + hash_c = %hash_c, + "Transitive resolution test passed" + ); + } + + #[tokio::test] + async fn cached_dependencies_are_reused() { + logging::init(); + + //* Given + let temp_dir = TempDir::new().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let mut fetcher = MockFetcher::new(); + + // Register a leaf manifest + let manifest = create_leaf_manifest("data"); + let manifest_hash = hash_canonical(&manifest).expect("should hash"); + fetcher.register("cached/dataset@1.0.0", manifest); + + let resolver = Resolver::new(fetcher, cache.clone()); + + let mut deps: BTreeMap = BTreeMap::new(); + deps.insert( + "cached".parse().unwrap(), + "cached/dataset@1.0.0".parse().unwrap(), + ); + + //* When + // First resolution - should populate cache + let graph1: Result = resolver.resolve_all(&deps).await; + + //* Then + assert!(graph1.is_ok(), "first resolution should succeed"); + assert!( + cache.contains(&manifest_hash), + "cache should contain manifest after first resolution" + ); + + // Verify the manifest can be retrieved from cache + let cached_manifest = cache.get(&manifest_hash).expect("should read cache"); + assert!(cached_manifest.is_some(), "cache should have manifest"); + + tracing::info!( + hash = %manifest_hash, + "Cache population test completed" + ); + } + + #[tokio::test] + async fn resolve_diamond_dependency_deduplicates() { + logging::init(); + + //* Given + let temp_dir = TempDir::new().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let mut fetcher = MockFetcher::new(); + + // Diamond: A -> B, A -> C, B -> D, C -> D + // D is shared between B and C + + let manifest_d = create_leaf_manifest("d_table"); + let hash_d = hash_canonical(&manifest_d).expect("should hash d"); + fetcher.register("ns/d@1.0.0", manifest_d); + + let manifest_b = create_manifest_with_dep("b_table", "d_dep", "ns/d@1.0.0"); + let hash_b = hash_canonical(&manifest_b).expect("should hash b"); + fetcher.register("ns/b@1.0.0", manifest_b); + + let manifest_c = create_manifest_with_dep("c_table", "d_dep", "ns/d@1.0.0"); + let hash_c = hash_canonical(&manifest_c).expect("should hash c"); + fetcher.register("ns/c@1.0.0", manifest_c); + + // A depends on both B and C + let manifest_a = serde_json::json!({ + "kind": "manifest", + "dependencies": { + "b_dep": "ns/b@1.0.0", + "c_dep": "ns/c@1.0.0" + }, + "tables": { + "a_table": { + "input": { "sql": "CREATE TABLE a_table AS SELECT 1" }, + "schema": { "arrow": { "fields": [] } }, + "network": "mainnet" + } + }, + "functions": {} + }); + let hash_a = hash_canonical(&manifest_a).expect("should hash a"); + fetcher.register("ns/a@1.0.0", manifest_a); + + let resolver = Resolver::new(fetcher, cache.clone()); + + let mut deps: BTreeMap = BTreeMap::new(); + deps.insert("a".parse().unwrap(), "ns/a@1.0.0".parse().unwrap()); + + //* When + let graph: Result = resolver.resolve_all(&deps).await; + + //* Then + let graph = graph.expect("resolution should succeed"); + + // D should appear only once despite being referenced by both B and C + assert_eq!(graph.len(), 4, "should have 4 unique dependencies"); + + // Verify all are cached + assert!(cache.contains(&hash_a)); + assert!(cache.contains(&hash_b)); + assert!(cache.contains(&hash_c)); + assert!(cache.contains(&hash_d)); + + // Verify topological order: D before B and C, B and C before A + let order = graph.topological_order().expect("should compute order"); + let pos_a = order.iter().position(|h| h == &hash_a).unwrap(); + let pos_b = order.iter().position(|h| h == &hash_b).unwrap(); + let pos_c = order.iter().position(|h| h == &hash_c).unwrap(); + let pos_d = order.iter().position(|h| h == &hash_d).unwrap(); + + assert!(pos_d < pos_b, "D should come before B"); + assert!(pos_d < pos_c, "D should come before C"); + assert!(pos_b < pos_a, "B should come before A"); + assert!(pos_c < pos_a, "C should come before A"); + + tracing::info!("Diamond dependency test passed"); + } + + #[tokio::test] + async fn resolve_missing_dependency_returns_error() { + logging::init(); + + //* Given + let temp_dir = TempDir::new().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let fetcher = MockFetcher::new(); + // Don't register any manifests + + let resolver = Resolver::new(fetcher, cache); + + let mut deps: BTreeMap = BTreeMap::new(); + deps.insert( + "missing".parse().unwrap(), + "missing/dataset@1.0.0".parse().unwrap(), + ); + + //* When + let result: Result = resolver.resolve_all(&deps).await; + + //* Then + assert!(result.is_err(), "should fail for missing dependency"); + let err = result.expect_err("should have error"); + assert!( + matches!(err, ResolveError::NotFound { .. }), + "should be NotFound error, got: {:?}", + err + ); + } + + #[tokio::test] + async fn locked_mode_validates_root() { + use dataset_authoring::lockfile::{Lockfile, LockfileError, RootInfo}; + + logging::init(); + + //* Given + let temp_dir = TempDir::new().expect("should create temp dir"); + let cache = Cache::with_root(temp_dir.path().to_path_buf()); + let mut fetcher = MockFetcher::new(); + + // Register a leaf manifest + let manifest = create_leaf_manifest("data"); + fetcher.register("source/dataset@1.0.0", manifest); + + let resolver = Resolver::new(fetcher, cache); + + let mut deps: BTreeMap = BTreeMap::new(); + deps.insert( + "source".parse().unwrap(), + "source/dataset@1.0.0".parse().unwrap(), + ); + + // Resolve and create lockfile with original root info + let graph = resolver.resolve_all(&deps).await.expect("should resolve"); + let original_root = RootInfo { + namespace: "original_ns".parse().expect("valid namespace"), + name: "original_dataset".parse().expect("valid name"), + version: "1.0.0".parse().expect("valid version"), + }; + let lockfile = Lockfile::from_graph(original_root.clone(), &graph); + + //* When - verify with matching root + let matching_result = lockfile.verify_with_root(Some(&original_root), &graph); + + //* Then + assert!( + matching_result.is_ok(), + "verification should succeed with matching root" + ); + + //* When - verify with different namespace + let different_ns_root = RootInfo { + namespace: "different_ns".parse().expect("valid namespace"), + name: "original_dataset".parse().expect("valid name"), + version: "1.0.0".parse().expect("valid version"), + }; + let mismatch_ns_result = lockfile.verify_with_root(Some(&different_ns_root), &graph); + + //* Then + assert!( + mismatch_ns_result.is_err(), + "verification should fail with different namespace" + ); + let err = mismatch_ns_result.expect_err("should have error"); + assert!( + matches!(err, LockfileError::RootMismatch { .. }), + "should be RootMismatch error, got: {:?}", + err + ); + + //* When - verify with different version + let different_version_root = RootInfo { + namespace: "original_ns".parse().expect("valid namespace"), + name: "original_dataset".parse().expect("valid name"), + version: "2.0.0".parse().expect("valid version"), + }; + let mismatch_version_result = + lockfile.verify_with_root(Some(&different_version_root), &graph); + + //* Then + assert!( + mismatch_version_result.is_err(), + "verification should fail with different version" + ); + let err = mismatch_version_result.expect_err("should have error"); + assert!( + matches!(err, LockfileError::RootMismatch { .. }), + "should be RootMismatch error, got: {:?}", + err + ); + + tracing::info!("Locked mode validates root test passed"); + } +} diff --git a/tests/src/tests/mod.rs b/tests/src/tests/mod.rs index 7891bda01..9c987fd7b 100644 --- a/tests/src/tests/mod.rs +++ b/tests/src/tests/mod.rs @@ -6,6 +6,7 @@ mod it_admin_api_jobs_progress; mod it_admin_api_schema; mod it_ampctl_gen_manifest; mod it_client; +mod it_dataset_authoring; mod it_dependencies; mod it_dump; mod it_functions;