diff --git a/.gitignore b/.gitignore index 3e3dfe2..32a3aae 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,8 @@ backend/.env.local backend/.env.*.local backend/*.sqlite backend/*.db -!backend/package-lock.json \ No newline at end of file +!backend/package-lock.json + +#storage-api +storage-api/storage-dev/ +!storage-api/package-lock.json \ No newline at end of file diff --git a/backend/migrations/20260923192128-create-dataset-registry.js b/backend/migrations/20260923192128-create-dataset-registry.js new file mode 100644 index 0000000..f2797fd --- /dev/null +++ b/backend/migrations/20260923192128-create-dataset-registry.js @@ -0,0 +1,64 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // Permanent record for one logical dataset. `internal_id` is the stable + // identity for the dataset's whole lifetime and is also the sandbox CouchDB + // `_id`. `dataset_id` (final public id) is assigned only at promotion. + async up(queryInterface, Sequelize) { + // Sequence backing NeuroJSON-generated public ids (njds######). Consumed + // only at promotion when the user did not request a valid custom id. + await queryInterface.sequelize.query( + "CREATE SEQUENCE IF NOT EXISTS neurojson_dataset_seq START 1;" + ); + + await queryInterface.createTable("dataset_registry", { + internal_id: { + type: Sequelize.UUID, + primaryKey: true, + allowNull: false, // app-generated (crypto.randomUUID); also the sandbox _id + }, + dataset_id: { + type: Sequelize.STRING(255), + allowNull: true, + unique: true, // final public id, set at promotion (njds###### or custom) + }, + requested_dataset_id: { + type: Sequelize.STRING(63), + allowNull: true, // user's preferred public id — a preference until promotion + }, + dataset_name: { + type: Sequelize.STRING(255), + allowNull: true, // canonical logical-dataset name + }, + owner_user_id: { + type: Sequelize.INTEGER, + allowNull: false, + references: { model: "users", key: "id" }, + onDelete: "RESTRICT", // don't orphan a dataset's owner + onUpdate: "CASCADE", + }, + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + updated_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + + await queryInterface.addIndex("dataset_registry", ["owner_user_id"], { + name: "idx_registry_owner", + }); + }, + + async down(queryInterface) { + await queryInterface.dropTable("dataset_registry"); + await queryInterface.sequelize.query( + "DROP SEQUENCE IF EXISTS neurojson_dataset_seq;" + ); + }, +}; diff --git a/backend/migrations/20260923192138-create-submissions.js b/backend/migrations/20260923192138-create-submissions.js new file mode 100644 index 0000000..5ee8bf9 --- /dev/null +++ b/backend/migrations/20260923192138-create-submissions.js @@ -0,0 +1,75 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // One review workflow (cycle) for a logical dataset. Holds only the current + // status + review/promotion metadata; the reviewer<->submitter discussion + // lives in submission_comments. A dataset can have many cycles over time, + // but only ONE open one (see the partial unique index below). + async up(queryInterface, Sequelize) { + await queryInterface.createTable("submissions", { + id: { + type: Sequelize.BIGINT, + autoIncrement: true, + primaryKey: true, + allowNull: false, + }, + submission_id: { + type: Sequelize.UUID, + allowNull: false, + unique: true, // one review cycle; referenced by submission_comments + }, + dataset_registry_id: { + type: Sequelize.UUID, + allowNull: false, + references: { model: "dataset_registry", key: "internal_id" }, + onDelete: "CASCADE", // removing the logical dataset removes its cycles + onUpdate: "CASCADE", + }, + status: { + type: Sequelize.STRING(50), + allowNull: false, + defaultValue: "pending", // pending|changes_requested|approved|rejected|promoted + }, + reviewed_by: { + type: Sequelize.INTEGER, + allowNull: true, + references: { model: "users", key: "id" }, + onDelete: "SET NULL", // keep the cycle if a reviewer is removed + onUpdate: "CASCADE", + }, + reviewed_at: { type: Sequelize.DATE, allowNull: true }, + promoted_db: { type: Sequelize.STRING(255), allowNull: true }, + promoted_at: { type: Sequelize.DATE, allowNull: true }, + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + updated_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + + await queryInterface.addIndex("submissions", ["dataset_registry_id"], { + name: "idx_submissions_registry", + }); + await queryInterface.addIndex("submissions", ["status"], { + name: "idx_submissions_status", + }); + + // At most one OPEN workflow per dataset. "Open" = still active: + // pending / changes_requested / approved (approved is awaiting promotion, + // so it must also block a new cycle). rejected/promoted are terminal, so a + // confirmed resubmit can insert a fresh pending row. + await queryInterface.sequelize.query( + "CREATE UNIQUE INDEX uq_one_open_per_dataset ON submissions (dataset_registry_id) WHERE status IN ('pending','changes_requested','approved');" + ); + }, + + async down(queryInterface) { + await queryInterface.dropTable("submissions"); + }, +}; diff --git a/backend/migrations/20260923192147-create-submission-comments.js b/backend/migrations/20260923192147-create-submission-comments.js new file mode 100644 index 0000000..c2e6b72 --- /dev/null +++ b/backend/migrations/20260923192147-create-submission-comments.js @@ -0,0 +1,50 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // Chronological reviewer <-> submitter discussion for a submission cycle, + // one row per comment. This is the single source of review discussion + // (there is no review_note column). Author role is derived at read time + // (user_id == dataset_registry.owner_user_id => submitter, else reviewer). + async up(queryInterface, Sequelize) { + await queryInterface.createTable("submission_comments", { + id: { + type: Sequelize.BIGINT, + autoIncrement: true, + primaryKey: true, + allowNull: false, + }, + submission_id: { + type: Sequelize.UUID, + allowNull: false, + references: { model: "submissions", key: "submission_id" }, + onDelete: "CASCADE", // comments die with their cycle + onUpdate: "CASCADE", + }, + user_id: { + type: Sequelize.INTEGER, + allowNull: false, + references: { model: "users", key: "id" }, + onDelete: "RESTRICT", // a comment must keep a real author + onUpdate: "CASCADE", + }, + message: { type: Sequelize.TEXT, allowNull: false }, + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + + // Serves the read query: WHERE submission_id = ? ORDER BY created_at ASC. + await queryInterface.addIndex( + "submission_comments", + ["submission_id", "created_at"], + { name: "idx_comments_submission_created" } + ); + }, + + async down(queryInterface) { + await queryInterface.dropTable("submission_comments"); + }, +}; diff --git a/backend/migrations/20260923214452-add-unique-requested-dataset-id.js b/backend/migrations/20260923214452-add-unique-requested-dataset-id.js new file mode 100644 index 0000000..de2ad8d --- /dev/null +++ b/backend/migrations/20260923214452-add-unique-requested-dataset-id.js @@ -0,0 +1,19 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // At most one dataset may reserve a given preferred public id (first-come). + // Partial so the many NULL (no preference) rows don't collide. Backs up the + // controller's assertRequestedIdFree check against races. + async up(queryInterface) { + await queryInterface.sequelize.query( + "CREATE UNIQUE INDEX uq_requested_dataset_id ON dataset_registry (requested_dataset_id) WHERE requested_dataset_id IS NOT NULL;" + ); + }, + + async down(queryInterface) { + await queryInterface.sequelize.query( + "DROP INDEX IF EXISTS uq_requested_dataset_id;" + ); + } +}; diff --git a/backend/migrations/20260925162336-add-requested-db-to-dataset-registry.js b/backend/migrations/20260925162336-add-requested-db-to-dataset-registry.js new file mode 100644 index 0000000..c760d73 --- /dev/null +++ b/backend/migrations/20260925162336-add-requested-db-to-dataset-registry.js @@ -0,0 +1,19 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // The public database a dataset should be published to at promotion. + // A preference/reservation; defaults to "public" when the user picks nothing. + // Actual db is resolved/created at promotion (a new name is review-gated). + async up(queryInterface, Sequelize) { + await queryInterface.addColumn("dataset_registry", "requested_db", { + type: Sequelize.STRING(63), + allowNull: false, + defaultValue: "public", + }); + }, + + async down(queryInterface) { + await queryInterface.removeColumn("dataset_registry", "requested_db"); + } +}; diff --git a/backend/migrations/20260928210352-add-draft-workflow-to-submissions.js b/backend/migrations/20260928210352-add-draft-workflow-to-submissions.js new file mode 100644 index 0000000..70f1725 --- /dev/null +++ b/backend/migrations/20260928210352-add-draft-workflow-to-submissions.js @@ -0,0 +1,75 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // Draft workflow: review status starts at 'draft'; JSON upload state is + // tracked separately from review state; raw ZIP is optional per submission. + async up(queryInterface, Sequelize) { + // New cycles start as draft (not pending). + await queryInterface.changeColumn("submissions", "status", { + type: Sequelize.STRING(50), + allowNull: false, + defaultValue: "draft", // draft|pending|changes_requested|approved|rejected|promoted + }); + + // JSON component state (separate from review status). + await queryInterface.addColumn("submissions", "json_status", { + type: Sequelize.STRING(20), + allowNull: false, + defaultValue: "uploaded", // uploaded | failed + }); + await queryInterface.addColumn("submissions", "json_uploaded_at", { + type: Sequelize.DATE, + allowNull: true, + }); + await queryInterface.addColumn("submissions", "json_error", { + type: Sequelize.TEXT, + allowNull: true, + }); + + // Last time the user sent it to review (draft/changes_requested → pending). + await queryInterface.addColumn("submissions", "submitted_at", { + type: Sequelize.DATE, + allowNull: true, + }); + + // Raw ZIP is optional; required for submit only when this is true. + await queryInterface.addColumn("submissions", "raw_zip_expected", { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: false, + }); + + // Existing rows already had their JSON uploaded. + await queryInterface.sequelize.query( + "UPDATE submissions SET json_uploaded_at = updated_at WHERE json_uploaded_at IS NULL;" + ); + + // One active cycle per dataset — now including draft. + await queryInterface.sequelize.query( + "DROP INDEX IF EXISTS uq_one_open_per_dataset;" + ); + await queryInterface.sequelize.query( + "CREATE UNIQUE INDEX uq_one_open_per_dataset ON submissions (dataset_registry_id) WHERE status IN ('draft','pending','changes_requested','approved');" + ); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.sequelize.query( + "DROP INDEX IF EXISTS uq_one_open_per_dataset;" + ); + await queryInterface.sequelize.query( + "CREATE UNIQUE INDEX uq_one_open_per_dataset ON submissions (dataset_registry_id) WHERE status IN ('pending','changes_requested','approved');" + ); + await queryInterface.removeColumn("submissions", "raw_zip_expected"); + await queryInterface.removeColumn("submissions", "submitted_at"); + await queryInterface.removeColumn("submissions", "json_error"); + await queryInterface.removeColumn("submissions", "json_uploaded_at"); + await queryInterface.removeColumn("submissions", "json_status"); + await queryInterface.changeColumn("submissions", "status", { + type: Sequelize.STRING(50), + allowNull: false, + defaultValue: "pending", + }); + }, +}; diff --git a/backend/migrations/20260930193225-create-raw-objects.js b/backend/migrations/20260930193225-create-raw-objects.js new file mode 100644 index 0000000..78dc49e --- /dev/null +++ b/backend/migrations/20260930193225-create-raw-objects.js @@ -0,0 +1,103 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // One stored raw dataset: the uploaded ZIP (transport artifact) and, after a + // reviewer triggers extraction on Zodiac, its extracted file tree. Shared by + // any number of dataset versions via submission_raw_files. All keys are + // RELATIVE to the Zodiac UPLOAD_ROOT (never absolute paths): + // /raw//source.zip + // /raw//files + // /raw//manifest.json + // zip_storage_key is permanent provenance; zip_deleted_at says whether the + // ZIP still physically exists (a future retention rule may delete it). + async up(queryInterface, Sequelize) { + await queryInterface.createTable("raw_objects", { + raw_id: { + type: Sequelize.UUID, + primaryKey: true, + allowNull: false, // generated by REN when it accepts a verified upload + }, + dataset_registry_id: { + type: Sequelize.UUID, + allowNull: false, + references: { model: "dataset_registry", key: "internal_id" }, + onDelete: "RESTRICT", + onUpdate: "CASCADE", + }, + zip_storage_key: { + type: Sequelize.STRING(512), + allowNull: false, + unique: true, + }, + original_filename: { type: Sequelize.STRING(255), allowNull: true }, + size_bytes: { type: Sequelize.BIGINT, allowNull: false }, // of the ZIP + sha256: { type: Sequelize.CHAR(64), allowNull: false }, // of the ZIP; not unique + zip_deleted_at: { type: Sequelize.DATE, allowNull: true }, + + extraction_status: { + type: Sequelize.STRING(20), + allowNull: false, + defaultValue: "not_extracted", // not_extracted|extracting|extracted|failed + }, + extraction_requested_by: { + type: Sequelize.INTEGER, + allowNull: true, + references: { model: "users", key: "id" }, + onDelete: "SET NULL", + onUpdate: "CASCADE", + }, + extraction_started_at: { type: Sequelize.DATE, allowNull: true }, + extracted_storage_key: { + type: Sequelize.STRING(512), + allowNull: true, + unique: true, + }, + manifest_storage_key: { + type: Sequelize.STRING(512), + allowNull: true, + unique: true, + }, + extracted_file_count: { type: Sequelize.INTEGER, allowNull: true }, + extracted_size_bytes: { type: Sequelize.BIGINT, allowNull: true }, + extracted_at: { type: Sequelize.DATE, allowNull: true }, + extraction_error: { type: Sequelize.TEXT, allowNull: true }, // safe message only + + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + updated_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + + await queryInterface.sequelize.query(` + ALTER TABLE raw_objects + ADD CONSTRAINT chk_raw_objects_extraction_status + CHECK (extraction_status IN ('not_extracted','extracting','extracted','failed')), + ADD CONSTRAINT chk_raw_objects_extracted_complete + CHECK (extraction_status <> 'extracted' + OR (extracted_storage_key IS NOT NULL + AND manifest_storage_key IS NOT NULL + AND extracted_at IS NOT NULL)); + `); + + await queryInterface.addIndex("raw_objects", ["dataset_registry_id"], { + name: "idx_raw_objects_registry", + }); + // Finds extractions stuck in 'extracting' (reconciliation). + await queryInterface.addIndex( + "raw_objects", + ["extraction_status", "updated_at"], + { name: "idx_raw_objects_extraction_status" } + ); + }, + + async down(queryInterface) { + await queryInterface.dropTable("raw_objects"); + }, +}; diff --git a/backend/migrations/20260930193235-create-submission-raw-files.js b/backend/migrations/20260930193235-create-submission-raw-files.js new file mode 100644 index 0000000..cd631ca --- /dev/null +++ b/backend/migrations/20260930193235-create-submission-raw-files.js @@ -0,0 +1,53 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // Which raw object(s) a dataset version (submission) uses. Composite PK + // (submission_id, raw_id) so a version could reference several objects in + // the future. For now a version has at most ONE object, enforced by + // uq_submission_raw_files_one_per_submission — drop that index to allow + // several (a path→object rule, e.g. a mount_path column, is then needed). + // Several versions may point to the same object (carry-forward = a new + // mapping row, no file copy/link). Deleting a version removes only its + // mapping; a mapped object can't be deleted (RESTRICT). Code must ensure + // the object and the submission belong to the same dataset. + async up(queryInterface, Sequelize) { + await queryInterface.createTable("submission_raw_files", { + submission_id: { + type: Sequelize.UUID, + primaryKey: true, // composite PK part 1 + allowNull: false, + references: { model: "submissions", key: "submission_id" }, + onDelete: "CASCADE", + onUpdate: "CASCADE", + }, + raw_id: { + type: Sequelize.UUID, + primaryKey: true, // composite PK part 2 + allowNull: false, + references: { model: "raw_objects", key: "raw_id" }, + onDelete: "RESTRICT", + onUpdate: "CASCADE", + }, + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + + // Current rule: one raw object per version (drop to allow several). + await queryInterface.sequelize.query( + "CREATE UNIQUE INDEX uq_submission_raw_files_one_per_submission ON submission_raw_files (submission_id);" + ); + + // "Which versions use this object?" — for cleanup of unmapped objects. + await queryInterface.addIndex("submission_raw_files", ["raw_id"], { + name: "idx_submission_raw_files_raw", + }); + }, + + async down(queryInterface) { + await queryInterface.dropTable("submission_raw_files"); + }, +}; diff --git a/backend/migrations/20260930193244-create-raw-upload-attempts.js b/backend/migrations/20260930193244-create-raw-upload-attempts.js new file mode 100644 index 0000000..0d3a281 --- /dev/null +++ b/backend/migrations/20260930193244-create-raw-upload-attempts.js @@ -0,0 +1,92 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // One tus upload of a raw ZIP for a dataset version. Tracks the upload + // process only — the final size/sha256 live on raw_objects. raw_id is set + // when REN accepts the verified upload (same transaction that creates the + // raw_objects row and the submission_raw_files mapping); a repeated + // completion message sees raw_id already set and does nothing. + async up(queryInterface, Sequelize) { + await queryInterface.createTable("raw_upload_attempts", { + upload_id: { + type: Sequelize.UUID, + primaryKey: true, + allowNull: false, // = the tus upload id carried in the upload token + }, + submission_id: { + type: Sequelize.UUID, + allowNull: false, + references: { model: "submissions", key: "submission_id" }, + onDelete: "CASCADE", + onUpdate: "CASCADE", + }, + status: { + type: Sequelize.STRING(20), + allowNull: false, + defaultValue: "initiated", + // initiated|uploading|verifying|complete|failed|cancelled|expired + }, + original_filename: { type: Sequelize.STRING(255), allowNull: true }, + // tus Upload-Length; also used for the size cap, disk check, final check. + expected_size: { type: Sequelize.BIGINT, allowNull: false }, + error: { type: Sequelize.TEXT, allowNull: true }, + uploaded_by: { + type: Sequelize.INTEGER, + allowNull: false, + references: { model: "users", key: "id" }, + onDelete: "RESTRICT", + onUpdate: "CASCADE", + }, + raw_id: { + type: Sequelize.UUID, + allowNull: true, + unique: true, // one upload can never produce two objects + references: { model: "raw_objects", key: "raw_id" }, + onDelete: "SET NULL", + onUpdate: "CASCADE", + }, + last_checked_at: { type: Sequelize.DATE, allowNull: true }, + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + updated_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + completed_at: { type: Sequelize.DATE, allowNull: true }, + }); + + // Only a completed attempt may point to an object. One direction only: + // a completed attempt may later have raw_id = NULL if cleanup deletes its + // (unmapped) object — ON DELETE SET NULL must not violate this check. + await queryInterface.sequelize.query(` + ALTER TABLE raw_upload_attempts + ADD CONSTRAINT chk_raw_upload_attempts_status + CHECK (status IN ('initiated','uploading','verifying','complete','failed','cancelled','expired')), + ADD CONSTRAINT chk_raw_upload_attempts_raw_only_when_complete + CHECK (raw_id IS NULL OR status = 'complete'); + `); + + await queryInterface.addIndex("raw_upload_attempts", ["submission_id"], { + name: "idx_raw_upload_attempts_submission", + }); + // Reconciliation scans stale in-progress attempts. + await queryInterface.addIndex( + "raw_upload_attempts", + ["status", "updated_at"], + { name: "idx_raw_upload_attempts_status" } + ); + // At most one active upload per version. + await queryInterface.sequelize.query( + "CREATE UNIQUE INDEX uq_one_active_raw_upload ON raw_upload_attempts (submission_id) WHERE status IN ('initiated','uploading','verifying');" + ); + }, + + async down(queryInterface) { + await queryInterface.dropTable("raw_upload_attempts"); + }, +}; diff --git a/backend/scripts/generate-upload-keys.js b/backend/scripts/generate-upload-keys.js new file mode 100644 index 0000000..a0f3e11 --- /dev/null +++ b/backend/scripts/generate-upload-keys.js @@ -0,0 +1,37 @@ +// Generate a new Ed25519 key pair (upload/file-read tokens) and an HMAC secret +// (REN <-> Zodiac service calls). Prints .env lines; writes nothing to disk. +// Usage: node scripts/generate-upload-keys.js [kid] (default kid: dev-YYYY-MM) +const crypto = require("crypto"); + +const now = new Date(); +const kid = + process.argv[2] || + `dev-${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, "0")}`; +if (!/^[a-z0-9][a-z0-9_-]{2,40}$/.test(kid)) { + console.error("kid must be 3-41 chars: lowercase letters, digits, - or _"); + process.exit(1); +} + +const { privateKey, publicKey } = crypto.generateKeyPairSync("ed25519"); +const privPem = privateKey.export({ type: "pkcs8", format: "pem" }); +const pubPem = publicKey.export({ type: "spki", format: "pem" }); +const hmacSecret = crypto.randomBytes(32).toString("base64url"); + +// dotenv turns \n inside double quotes into real newlines. +const oneLine = (pem) => pem.trim().replace(/\n/g, "\\n"); + +console.log("# ---- REN backend/.env (keep private) ----"); +console.log(`UPLOAD_TOKEN_KID=${kid}`); +console.log(`UPLOAD_TOKEN_PRIVATE_KEY="${oneLine(privPem)}"`); +console.log(`STORAGE_HMAC_ACTIVE_KID=${kid}`); +console.log(`STORAGE_HMAC_KEYS='${JSON.stringify({ [kid]: hmacSecret })}'`); +console.log(""); +console.log("# ---- Zodiac storage-api/.env ----"); +console.log( + `UPLOAD_TOKEN_PUBLIC_KEYS='${JSON.stringify({ [kid]: pubPem.trim() })}'` +); +console.log(`STORAGE_HMAC_ACTIVE_KID=${kid}`); +console.log(`STORAGE_HMAC_KEYS='${JSON.stringify({ [kid]: hmacSecret })}'`); +console.log(""); +console.log("# Rotation: add the new kid to the Zodiac maps first, then switch REN"); +console.log("# to the new kid, then remove the old kid after 15+ minutes."); diff --git a/backend/scripts/mint-test-upload-token.js b/backend/scripts/mint-test-upload-token.js new file mode 100644 index 0000000..6592b36 --- /dev/null +++ b/backend/scripts/mint-test-upload-token.js @@ -0,0 +1,19 @@ +// DEV ONLY: print a signed tus-upload token for a test upload of bytes, +// with made-up ids (the storage API doesn't ask REN about uploads yet). +// Usage: node scripts/mint-test-upload-token.js +require("dotenv").config({ quiet: true }); +const crypto = require("crypto"); +const { signUploadToken } = require("../src/lib/uploadTokens"); + +const size = Number(process.argv[2]); +if (!Number.isSafeInteger(size) || size <= 0) { + console.error("usage: node scripts/mint-test-upload-token.js "); + process.exit(1); +} +const ids = { + uploadId: crypto.randomUUID(), + internalId: crypto.randomUUID(), + submissionId: crypto.randomUUID(), +}; +const { token } = signUploadToken({ userId: 0, ...ids, expectedSize: size }); +console.log(JSON.stringify({ ...ids, size, token })); diff --git a/backend/scripts/reconcile-raw-once.js b/backend/scripts/reconcile-raw-once.js new file mode 100644 index 0000000..740aeea --- /dev/null +++ b/backend/scripts/reconcile-raw-once.js @@ -0,0 +1,10 @@ +// Run ONE raw-upload reconciliation pass now and print what happened. +// Usage: node scripts/reconcile-raw-once.js +require("dotenv").config({ quiet: true }); +const { reconcileOnce } = require("../src/jobs/reconcileRawUploads"); +const { sequelize } = require("../src/config/database"); + +reconcileOnce() + .then((s) => console.log("summary:", s)) + .catch((e) => console.error("failed:", e.message)) + .finally(() => sequelize.close()); diff --git a/backend/scripts/send-storage-event.js b/backend/scripts/send-storage-event.js new file mode 100644 index 0000000..e6e51b3 --- /dev/null +++ b/backend/scripts/send-storage-event.js @@ -0,0 +1,32 @@ +// Send a signed storage event to the local backend, as Zodiac would (testing). +// Usage: node scripts/send-storage-event.js [size] [sha256|reason] +require("dotenv").config({ quiet: true }); +const { signRequest } = require("../src/lib/storageSignature"); + +const [uploadId, event, size, extra] = process.argv.slice(2); +if (!uploadId || !event) { + console.log( + "usage: node scripts/send-storage-event.js [size] [sha256|reason]" + ); + process.exit(1); +} +const payload = { event }; +if (event === "verified" || event === "committed") { + Object.assign(payload, { size: Number(size), sha256: extra }); +} +if (event === "failed") payload.reason = size || "storage_error"; + +const base = process.env.MAIN_API_LOCAL_URL || "http://localhost:5000"; +const path = `/api/v1/internal/storage/uploads/${uploadId}/events`; +const body = JSON.stringify(payload); + +fetch(base + path, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...signRequest({ method: "POST", path, body }), + }, + body, +}) + .then(async (r) => console.log(r.status, await r.text())) + .catch((e) => console.log("request failed:", e.message)); diff --git a/backend/scripts/test-upload-signing.js b/backend/scripts/test-upload-signing.js new file mode 100644 index 0000000..634c636 --- /dev/null +++ b/backend/scripts/test-upload-signing.js @@ -0,0 +1,89 @@ +// Self-test for the upload token + service-signature helpers (prints no secrets). +// Usage: node scripts/test-upload-signing.js (reads backend/.env) +require("dotenv").config(); +const crypto = require("crypto"); +const { + signUploadToken, + verifyToken, + signToken, +} = require("../src/lib/uploadTokens"); +const { signRequest, verifyRequest } = require("../src/lib/storageSignature"); + +let failed = 0; +const check = (name, fn) => { + try { + fn(); + console.log("PASS", name); + } catch (e) { + failed++; + console.log("FAIL", name, "-", e.message); + } +}; +const expectThrow = (fn, msg) => { + try { + fn(); + } catch { + return; + } + throw new Error(`expected failure: ${msg}`); +}; + +const kid = process.env.UPLOAD_TOKEN_KID; +const pub = { + [kid]: crypto.createPublicKey(process.env.UPLOAD_TOKEN_PRIVATE_KEY), +}; +const { token } = signUploadToken({ + userId: 1, + uploadId: crypto.randomUUID(), + internalId: crypto.randomUUID(), + submissionId: crypto.randomUUID(), + expectedSize: 1234, +}); + +check("valid upload token verifies", () => { + const c = verifyToken(token, pub, "tus-upload"); + if (c.len !== 1234) throw new Error("len claim wrong"); +}); +check("tampered payload is rejected", () => { + const [h, p, s] = token.split("."); + const bad = JSON.parse(Buffer.from(p, "base64url").toString()); + bad.len = 999999; + const forged = `${h}.${Buffer.from(JSON.stringify(bad)).toString( + "base64url" + )}.${s}`; + expectThrow(() => verifyToken(forged, pub, "tus-upload"), "tamper"); +}); +check("wrong token type is rejected", () => + expectThrow(() => verifyToken(token, pub, "file-read"), "typ") +); +check("unknown kid is rejected", () => + expectThrow(() => verifyToken(token, {}, "tus-upload"), "kid") +); +check("expired token is rejected", () => { + const { token: old } = signToken("tus-upload", {}, -120); + expectThrow(() => verifyToken(old, pub, "tus-upload"), "exp"); +}); + +const body = JSON.stringify({ event: "verified" }); +const path = "/api/v1/internal/storage/uploads/x/events"; +const headers = signRequest({ method: "POST", path, body }); +check("valid HMAC request verifies", () => { + const r = verifyRequest({ method: "POST", path, headers, rawBody: body }); + if (!r.ok) throw new Error(r.reason); +}); +check("HMAC rejects changed body", () => { + const r = verifyRequest({ method: "POST", path, headers, rawBody: body + " " }); + if (r.ok) throw new Error("accepted"); +}); +check("HMAC rejects changed path", () => { + const r = verifyRequest({ + method: "POST", + path: path + "?x=1", + headers, + rawBody: body, + }); + if (r.ok) throw new Error("accepted"); +}); + +console.log(failed ? `\n${failed} check(s) FAILED` : "\nAll checks passed"); +process.exit(failed ? 1 : 0); diff --git a/backend/src/controllers/rawUploads.controller.js b/backend/src/controllers/rawUploads.controller.js new file mode 100644 index 0000000..f68df74 --- /dev/null +++ b/backend/src/controllers/rawUploads.controller.js @@ -0,0 +1,427 @@ +// Raw ZIP upload sessions (owner side). REN only authorizes and tracks state; +// the bytes go browser → Zodiac storage API via tus with a short-lived token. +// Completion (Zodiac events + raw_objects creation) is handled separately. +const crypto = require("crypto"); +const { sequelize } = require("../config/database"); +const { + HttpError, + UUID_RE, + loadOwnedRegistry, +} = require("../lib/uploadAccess"); +const { signUploadToken } = require("../lib/uploadTokens"); + +const MAX_UPLOAD_BYTES = Number( + process.env.RAW_MAX_UPLOAD_BYTES || 50000000000 +); // 50 GB (decimal, matches the size shown in the UI); sent to the page +const CHUNK_BYTES = Number(process.env.RAW_CHUNK_BYTES || 67108864); // 64 MB +const ACTIVE = ["initiated", "uploading", "verifying"]; +const EDITABLE = ["draft", "changes_requested"]; + +const sendError = (res, err, label) => { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + console.error(`${label} failed:`, err.message); + res.status(500).json({ error: `${label} failed.` }); +}; + +// Storage endpoint the browser uploads to (env only; differs dev vs prod). +const storageEndpoint = () => { + const base = process.env.STORAGE_PUBLIC_URL; + if (!base) { + throw new HttpError(503, { + error: "Raw data upload is not configured on this server.", + }); + } + return `${base.replace(/\/+$/, "")}/files`; +}; + +// Display filename only — never used for paths. Basename, trimmed, .zip. +const cleanFilename = (raw) => { + const name = String(raw || "") + .split(/[\\/]/) + .pop() + .trim(); + if (!name || name.length > 255) { + throw new HttpError(400, { error: "Invalid file name." }); + } + if (!/\.zip$/i.test(name)) { + throw new HttpError(400, { error: "Raw data must be a .zip file." }); + } + return name; +}; + +const cleanSize = (raw) => { + const size = Number(raw); + if (!Number.isSafeInteger(size) || size <= 0) { + throw new HttpError(400, { error: "Invalid file size." }); + } + if (size > MAX_UPLOAD_BYTES) { + throw new HttpError(413, { + code: "FILE_TOO_LARGE", + error: `The file is too large (max ${Math.floor( + MAX_UPLOAD_BYTES / 1e9 + )} GB).`, + }); + } + return size; +}; + +// Latest submission (cycle) of the dataset, locked for this transaction. +const lockLatestSubmission = async (internalId, t) => { + const rows = await sequelize.query( + `SELECT id, submission_id, status FROM submissions + WHERE dataset_registry_id = :iid ORDER BY created_at DESC LIMIT 1 FOR UPDATE`, + { + replacements: { iid: internalId }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (!rows[0]) throw new HttpError(404, { error: "No submission found." }); + return rows[0]; +}; + +const assertEditable = (sub) => { + if (!EDITABLE.includes(sub.status)) { + throw new HttpError(409, { + code: "NOT_EDITABLE", + error: + "Raw data can only be changed while the submission is a draft or has requested changes.", + }); + } +}; + +const tokenFor = (req, reg, sub, attempt) => + signUploadToken({ + userId: req.user.id, + uploadId: attempt.upload_id, + internalId: reg.internal_id, + submissionId: sub.submission_id, + expectedSize: Number(attempt.expected_size), // BIGINT comes back as a string + }); + +// POST /api/v1/uploads/:internalId/raw {filename, size} +// Start a raw ZIP upload, or resume the active one (same name + size). +const startRawUpload = async (req, res) => { + try { + const filename = cleanFilename(req.body?.filename); + const size = cleanSize(req.body?.size); + const endpoint = storageEndpoint(); + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + + const out = await sequelize.transaction(async (t) => { + const sub = await lockLatestSubmission(reg.internal_id, t); + assertEditable(sub); + + const active = await sequelize.query( + `SELECT upload_id, original_filename, expected_size FROM raw_upload_attempts + WHERE submission_id = :sid AND status IN (:active)`, + { + replacements: { sid: sub.submission_id, active: ACTIVE }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (active[0]) { + const a = active[0]; + if ( + a.original_filename === filename && + Number(a.expected_size) === size + ) { + return { sub, attempt: a, resumed: true }; // same file → resume + } + throw new HttpError(409, { + code: "ACTIVE_UPLOAD_EXISTS", + error: + "Another raw data upload is in progress. Resume it with the same file, or cancel it first.", + upload: { + uploadId: a.upload_id, + filename: a.original_filename, + size: Number(a.expected_size), + }, + }); + } + + const attempt = { + upload_id: crypto.randomUUID(), + original_filename: filename, + expected_size: size, + }; + await sequelize.query( + `INSERT INTO raw_upload_attempts + (upload_id, submission_id, status, original_filename, expected_size, uploaded_by, created_at, updated_at) + VALUES (:uid, :sid, 'initiated', :name, :size, :by, NOW(), NOW())`, + { + replacements: { + uid: attempt.upload_id, + sid: sub.submission_id, + name: filename, + size, + by: req.user.id, + }, + transaction: t, + } + ); + // Starting a raw upload means this version includes raw data. + await sequelize.query( + `UPDATE submissions SET raw_zip_expected = true, updated_at = NOW() WHERE id = :id`, + { replacements: { id: sub.id }, transaction: t } + ); + return { sub, attempt, resumed: false }; + }); + + const { token, expiresAt } = tokenFor(req, reg, out.sub, out.attempt); + res.status(out.resumed ? 200 : 201).json({ + uploadId: out.attempt.upload_id, + endpoint, + token, + expiresAt, + chunkSize: CHUNK_BYTES, + resumed: out.resumed, + }); + } catch (err) { + // Unique index backstop: two starts at the same moment. + if (err.original?.code === "23505") { + return res.status(409).json({ + code: "ACTIVE_UPLOAD_EXISTS", + error: "Another raw data upload is in progress.", + }); + } + sendError(res, err, "Start raw upload"); + } +}; + +// Load an attempt that belongs to this dataset (owner already checked). +const loadAttempt = async (reg, uploadId, t) => { + if (!UUID_RE.test(uploadId)) { + throw new HttpError(404, { error: "Upload not found." }); + } + const rows = await sequelize.query( + `SELECT a.upload_id, a.status, a.expected_size, a.original_filename, + s.submission_id, s.status AS submission_status + FROM raw_upload_attempts a + JOIN submissions s ON s.submission_id = a.submission_id + WHERE a.upload_id = :uid AND s.dataset_registry_id = :iid`, + { + replacements: { uid: uploadId, iid: reg.internal_id }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (!rows[0]) throw new HttpError(404, { error: "Upload not found." }); + return rows[0]; +}; + +// POST /api/v1/uploads/:internalId/raw/:uploadId/token — fresh token for a +// long-running upload. +const refreshRawUploadToken = async (req, res) => { + try { + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + const a = await loadAttempt(reg, String(req.params.uploadId)); + if (!ACTIVE.includes(a.status)) { + throw new HttpError(409, { + code: "UPLOAD_NOT_ACTIVE", + error: "This upload is no longer active.", + }); + } + if (!EDITABLE.includes(a.submission_status)) { + throw new HttpError(409, { + code: "NOT_EDITABLE", + error: "The submission can no longer be changed.", + }); + } + const { token, expiresAt } = tokenFor( + req, + reg, + { submission_id: a.submission_id }, + a + ); + res.json({ token, expiresAt }); + } catch (err) { + sendError(res, err, "Refresh upload token"); + } +}; + +// GET /api/v1/uploads/:internalId/raw — current raw object + active/last attempt. +const getRawStatus = async (req, res) => { + try { + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + const subs = await sequelize.query( + `SELECT submission_id, status, raw_zip_expected FROM submissions + WHERE dataset_registry_id = :iid ORDER BY created_at DESC LIMIT 1`, + { replacements: { iid: reg.internal_id }, type: sequelize.QueryTypes.SELECT } + ); + const sub = subs[0]; + if (!sub) { + return res.json({ + rawZipExpected: false, + editable: false, + maxUploadBytes: MAX_UPLOAD_BYTES, + current: null, + activeUpload: null, + lastUpload: null, + }); + } + + const current = await sequelize.query( + `SELECT o.raw_id, o.original_filename, o.size_bytes, o.sha256, + o.extraction_status, o.created_at, + -- carried forward = the upload that made this object belongs + -- to a different version (exact; not a timestamp guess) + (a.submission_id IS NOT NULL AND a.submission_id <> f.submission_id) + AS carried_forward + FROM submission_raw_files f + JOIN raw_objects o ON o.raw_id = f.raw_id + LEFT JOIN raw_upload_attempts a ON a.raw_id = o.raw_id + WHERE f.submission_id = :sid`, + { + replacements: { sid: sub.submission_id }, + type: sequelize.QueryTypes.SELECT, + } + ); + const attempts = await sequelize.query( + `SELECT upload_id, status, original_filename, expected_size, error, + created_at, updated_at, completed_at + FROM raw_upload_attempts WHERE submission_id = :sid + ORDER BY created_at DESC LIMIT 1`, + { + replacements: { sid: sub.submission_id }, + type: sequelize.QueryTypes.SELECT, + } + ); + const last = attempts[0] || null; + const shape = (a) => + a && { + uploadId: a.upload_id, + status: a.status, + filename: a.original_filename, + size: Number(a.expected_size), + error: a.error || null, + createdAt: a.created_at, + updatedAt: a.updated_at, + completedAt: a.completed_at, + }; + const c = current[0]; + res.json({ + rawZipExpected: !!sub.raw_zip_expected, + editable: EDITABLE.includes(sub.status), + maxUploadBytes: MAX_UPLOAD_BYTES, + // Storage keys stay server-side; only display facts are returned. + current: c + ? { + rawId: c.raw_id, + filename: c.original_filename, + size: Number(c.size_bytes), + sha256: c.sha256, + extractionStatus: c.extraction_status, + carriedForward: !!c.carried_forward, + uploadedAt: c.created_at, + } + : null, + activeUpload: last && ACTIVE.includes(last.status) ? shape(last) : null, + // "Last upload failed/cancelled/expired" (the current ZIP is untouched). + lastUpload: + last && ["failed", "cancelled", "expired"].includes(last.status) + ? shape(last) + : null, + }); + } catch (err) { + sendError(res, err, "Load raw data status"); + } +}; + +// DELETE /api/v1/uploads/:internalId/raw/:uploadId — cancel an active upload. +// One conditional UPDATE, so it can't race the completion step: if REN has +// already committed it, the row is 'complete' and this returns 409. +const cancelRawUpload = async (req, res) => { + try { + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + const a = await loadAttempt(reg, String(req.params.uploadId)); + const rows = await sequelize.query( + `UPDATE raw_upload_attempts + SET status = 'cancelled', completed_at = NOW(), updated_at = NOW() + WHERE upload_id = :uid AND status IN (:active) + RETURNING upload_id`, + { + replacements: { uid: a.upload_id, active: ACTIVE }, + type: sequelize.QueryTypes.SELECT, + } + ); + if (!rows.length) { + throw new HttpError(409, { + code: "UPLOAD_NOT_ACTIVE", + error: "This upload has already finished.", + }); + } + // Zodiac's partial data is removed by the reconciliation/cleanup step; + // the browser also sends a tus DELETE when the user cancels. + res.json({ status: "cancelled" }); + } catch (err) { + sendError(res, err, "Cancel raw upload"); + } +}; + +// PATCH /api/v1/uploads/:internalId/raw/expected {expected: boolean} +// "This dataset includes raw data". Can only be turned off when there is no +// raw ZIP and no active upload. +const setRawExpected = async (req, res) => { + try { + if (typeof req.body?.expected !== "boolean") { + throw new HttpError(400, { error: "`expected` must be true or false." }); + } + const expected = req.body.expected; + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + await sequelize.transaction(async (t) => { + const sub = await lockLatestSubmission(reg.internal_id, t); + assertEditable(sub); + if (!expected) { + const [{ busy }] = await sequelize.query( + `SELECT (EXISTS (SELECT 1 FROM submission_raw_files WHERE submission_id = :sid) + OR EXISTS (SELECT 1 FROM raw_upload_attempts + WHERE submission_id = :sid AND status IN (:active))) AS busy`, + { + replacements: { sid: sub.submission_id, active: ACTIVE }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (busy) { + throw new HttpError(409, { + code: "RAW_DATA_PRESENT", + error: + "This version already has raw data (or an upload in progress), so it can't be unmarked.", + }); + } + } + await sequelize.query( + `UPDATE submissions SET raw_zip_expected = :e, updated_at = NOW() WHERE id = :id`, + { replacements: { e: expected, id: sub.id }, transaction: t } + ); + }); + res.json({ rawZipExpected: expected }); + } catch (err) { + sendError(res, err, "Update raw data setting"); + } +}; + +module.exports = { + startRawUpload, + refreshRawUploadToken, + getRawStatus, + cancelRawUpload, + setRawExpected, +}; diff --git a/backend/src/controllers/storageEvents.controller.js b/backend/src/controllers/storageEvents.controller.js new file mode 100644 index 0000000..533e148 --- /dev/null +++ b/backend/src/controllers/storageEvents.controller.js @@ -0,0 +1,229 @@ +// Events from the Zodiac storage API about one raw ZIP upload. Idempotent and +// forward-only: Zodiac may resend any event (outbox retries, reconciliation). +// started → initiated → uploading +// verified → REN decides commit | discard +// committed → one transaction: raw_objects + attempt complete + mapping +// failed → attempt failed (safe message) +// raw_id = upload_id (deterministic, so a repeated "verified" gets the same +// answer and Zodiac always moves the file to the same path). +const { sequelize } = require("../config/database"); +const { HttpError, UUID_RE } = require("../lib/uploadAccess"); + +const ACTIVE = ["initiated", "uploading", "verifying"]; +const EDITABLE = ["draft", "changes_requested"]; +const SHA_RE = /^[0-9a-f]{64}$/; + +// Zodiac sends a reason code; users only ever see these fixed sentences. +const FAILURE_MESSAGES = { + not_zip: "The file is not a ZIP archive.", + size_mismatch: "The uploaded file size did not match the expected size.", + storage_error: "The storage server could not save the file.", +}; +const failureMessage = (code) => + FAILURE_MESSAGES[code] || "The upload could not be completed."; + +const zipKey = (internalId, rawId) => `${internalId}/raw/${rawId}/source.zip`; + +// Lock the attempt row and read its submission + dataset. +const lockAttempt = async (uploadId, t) => { + const rows = await sequelize.query( + `SELECT a.upload_id, a.status, a.expected_size, a.original_filename, a.raw_id, + s.id AS submission_pk, s.submission_id, s.status AS submission_status, + s.dataset_registry_id + FROM raw_upload_attempts a + JOIN submissions s ON s.submission_id = a.submission_id + WHERE a.upload_id = :uid + FOR UPDATE OF a`, + { + replacements: { uid: uploadId }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (!rows[0]) throw new HttpError(404, { error: "Unknown upload." }); + return rows[0]; +}; + +const markFailed = (uploadId, message, t) => + sequelize.query( + `UPDATE raw_upload_attempts + SET status = 'failed', error = :msg, completed_at = NOW(), updated_at = NOW() + WHERE upload_id = :uid AND status IN (:active)`, + { + replacements: { uid: uploadId, msg: message, active: ACTIVE }, + transaction: t, + } + ); + +// size + sha256 sent by Zodiac must match what this upload expected. +const checkFile = (a, body) => { + const size = Number(body.size); + if (!Number.isSafeInteger(size) || !SHA_RE.test(String(body.sha256 || ""))) { + throw new HttpError(400, { error: "size and sha256 are required." }); + } + return { + size, + sha256: body.sha256, + sizeOk: size === Number(a.expected_size), + }; +}; + +const onStarted = async (a, body, t) => { + await sequelize.query( + `UPDATE raw_upload_attempts SET status = 'uploading', updated_at = NOW() + WHERE upload_id = :uid AND status = 'initiated'`, + { replacements: { uid: a.upload_id }, transaction: t } + ); + return { ok: true }; +}; + +const onVerified = async (a, body, t) => { + if (a.status === "complete") { + // Already committed earlier — repeat the same answer. + return { + decision: "commit", + rawId: a.raw_id, + storageKey: zipKey(a.dataset_registry_id, a.raw_id), + }; + } + if (!ACTIVE.includes(a.status)) { + // cancelled / failed / expired + return { decision: "discard", reason: `upload is ${a.status}` }; + } + const { sizeOk } = checkFile(a, body); + if (!sizeOk) { + await markFailed(a.upload_id, failureMessage("size_mismatch"), t); + return { decision: "discard", reason: "size_mismatch" }; + } + if (!EDITABLE.includes(a.submission_status)) { + await markFailed( + a.upload_id, + "The submission was locked before the upload finished.", + t + ); + return { decision: "discard", reason: "submission_locked" }; + } + await sequelize.query( + `UPDATE raw_upload_attempts SET status = 'verifying', updated_at = NOW() + WHERE upload_id = :uid AND status IN ('initiated','uploading')`, + { replacements: { uid: a.upload_id }, transaction: t } + ); + const rawId = a.upload_id; // deterministic: same answer every time + return { + decision: "commit", + rawId, + storageKey: zipKey(a.dataset_registry_id, rawId), + }; +}; + +const onCommitted = async (a, body, t) => { + const { size, sha256, sizeOk } = checkFile(a, body); + const rawId = a.upload_id; + + if (a.status === "complete") { + // Repeat: confirm it's the same file, change nothing. + const [obj] = await sequelize.query( + `SELECT sha256 FROM raw_objects WHERE raw_id = :rid`, + { + replacements: { rid: a.raw_id }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (obj && obj.sha256 !== sha256) { + throw new HttpError(409, { + error: "Upload already completed with a different file.", + }); + } + return { action: "keep" }; + } + if (!ACTIVE.includes(a.status)) { + // e.g. the user cancelled after REN said "commit" — Zodiac deletes the file. + return { action: "discard", reason: `upload is ${a.status}` }; + } + if (!sizeOk) { + await markFailed(a.upload_id, failureMessage("size_mismatch"), t); + return { action: "discard", reason: "size_mismatch" }; + } + + // ONE transaction: the object, the finished attempt, and the version mapping. + await sequelize.query( + `INSERT INTO raw_objects + (raw_id, dataset_registry_id, zip_storage_key, original_filename, + size_bytes, sha256, created_at, updated_at) + VALUES (:rid, :iid, :key, :name, :size, :sha, NOW(), NOW())`, + { + replacements: { + rid: rawId, + iid: a.dataset_registry_id, + key: zipKey(a.dataset_registry_id, rawId), + name: a.original_filename, + size, + sha: sha256, + }, + transaction: t, + } + ); + await sequelize.query( + `UPDATE raw_upload_attempts + SET status = 'complete', raw_id = :rid, error = NULL, + completed_at = NOW(), updated_at = NOW() + WHERE upload_id = :uid`, + { replacements: { rid: rawId, uid: a.upload_id }, transaction: t } + ); + // Replace (or create) this version's raw ZIP. The previous object, if any, + // becomes unmapped and is left for the cleanup job. + await sequelize.query( + `INSERT INTO submission_raw_files (submission_id, raw_id, created_at) + VALUES (:sid, :rid, NOW()) + ON CONFLICT (submission_id) DO UPDATE SET raw_id = EXCLUDED.raw_id, created_at = NOW()`, + { replacements: { sid: a.submission_id, rid: rawId }, transaction: t } + ); + await sequelize.query( + `UPDATE submissions SET raw_zip_expected = true, updated_at = NOW() WHERE id = :id`, + { replacements: { id: a.submission_pk }, transaction: t } + ); + return { action: "keep" }; +}; + +const onFailed = async (a, body, t) => { + await markFailed(a.upload_id, failureMessage(String(body.reason || "")), t); + return { ok: true }; +}; + +const HANDLERS = { + started: onStarted, + verified: onVerified, + committed: onCommitted, + failed: onFailed, +}; + +// Run one storage event for an upload (shared by the HTTP endpoint for +// pushes and by the reconciliation job for pulls). Returns REN's answer. +const processStorageEvent = async (uploadId, body = {}) => { + if (!UUID_RE.test(uploadId)) { + throw new HttpError(404, { error: "Unknown upload." }); + } + const handler = HANDLERS[body.event]; + if (!handler) throw new HttpError(400, { error: "Unknown event." }); + return sequelize.transaction(async (t) => { + const a = await lockAttempt(uploadId, t); + return handler(a, body, t); + }); +}; + +// POST /api/v1/internal/storage/uploads/:uploadId/events {event, ...} +const handleUploadEvent = async (req, res) => { + try { + res.json( + await processStorageEvent(String(req.params.uploadId), req.body || {}) + ); + } catch (err) { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + console.error("Storage event failed:", err.message); + // 5xx → Zodiac's outbox retries later. + res.status(500).json({ error: "Storage event failed." }); + } +}; + +module.exports = { handleUploadEvent, processStorageEvent }; diff --git a/backend/src/controllers/uploads.controller.js b/backend/src/controllers/uploads.controller.js new file mode 100644 index 0000000..96596b4 --- /dev/null +++ b/backend/src/controllers/uploads.controller.js @@ -0,0 +1,763 @@ +const axios = require("axios"); +const crypto = require("crypto"); +const { sequelize } = require("../config/database"); +const { HttpError, loadOwnedRegistry } = require("../lib/uploadAccess"); + +// Upload a JSON dataset into the sandbox CouchDB db for review, tracked in +// PostgreSQL. See migrations create-dataset-registry / create-submissions / +// create-submission-comments. +// +// Identity model: +// dataset_registry.internal_id = permanent logical-dataset id (UUID); also +// the sandbox CouchDB _id. Lives forever. +// dataset_registry.dataset_id = FINAL public id, assigned only at promotion +// (njds###### or the user's custom id). +// submissions.submission_id = one review cycle (UUID). +// The sandbox holds the latest working copy at _id = internal_id; a re-upload +// full-replaces it (the `timestamp` update handler on the db). +const UPLOAD_URL = process.env.COUCHDB_UPLOAD_URL || process.env.COUCHDB_URL; +const UPLOAD_DB = process.env.COUCHDB_UPLOAD_DB; + +const buildHeaders = () => { + const headers = { "Content-Type": "application/json" }; + const user = process.env.COUCHDB_UPLOAD_USER; + const pass = process.env.COUCHDB_UPLOAD_PASSWORD; + if (user && pass) { + headers.Authorization = + "Basic " + Buffer.from(`${user}:${pass}`).toString("base64"); + } + return headers; +}; + +// Preferred public id rules (final availability is re-checked at promotion). +const REQUESTED_ID_RE = /^[a-z0-9][a-z0-9_-]{2,62}$/; +const validateRequestedId = (id) => { + if (!REQUESTED_ID_RE.test(id)) { + throw new HttpError(400, { + error: + "Preferred dataset ID must be 3–63 characters: lowercase letters, digits, hyphen or underscore, and cannot start with a hyphen or underscore.", + }); + } + if (id.startsWith("njds")) { + throw new HttpError(400, { + error: + 'Preferred dataset ID cannot start with "njds" (reserved for NeuroJSON-assigned IDs).', + }); + } +}; + +// Target public database name (same slug rule; no njds restriction — that is +// only for dataset ids). Blank defaults to "public" upstream. +// CouchDB requires db names to start with a lowercase letter, so this is +// stricter than the dataset-id rule. +const validateRequestedDb = (db) => { + if (!/^[a-z][a-z0-9_-]{2,62}$/.test(db)) { + throw new HttpError(400, { + error: + "Database name must be 3–63 characters, start with a lowercase letter, and use only lowercase letters, numbers, - or _.", + }); + } +}; + +// Preliminary availability: reject an id already used as a final public +// dataset_id OR another row's requested_dataset_id. `excludeInternalId` skips +// the caller's own row on update. (The authoritative check still runs at +// promotion; a DB partial-unique index also backs this up.) +const assertRequestedIdFree = async (requestedId, excludeInternalId, t) => { + const rows = await sequelize.query( + `SELECT 1 FROM dataset_registry + WHERE (dataset_id = :id OR requested_dataset_id = :id) + AND (:exclude::uuid IS NULL OR internal_id <> :exclude::uuid) + LIMIT 1`, + { + replacements: { id: requestedId, exclude: excludeInternalId || null }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (rows.length) { + throw new HttpError(409, { + code: "REQUESTED_ID_TAKEN", + error: `The preferred ID "${requestedId}" is already in use.`, + }); + } +}; + +const createUpload = async (req, res) => { + // Existing editable cycle whose JSON update is being attempted; used to + // record a failure after the transaction rolls back. + let jsonTargetId = null; + try { + const body = req.body; + if (!body || typeof body !== "object" || Array.isArray(body)) { + return res + .status(400) + .json({ error: "Request body must be a JSON object" }); + } + + // Strip top-level CouchDB-reserved fields (_id, _rev, ...). + const clean = {}; + for (const [key, value] of Object.entries(body)) { + if (!key.startsWith("_")) clean[key] = value; + } + + // dataset name: file's dataset_description.json.Name → ?datasetName → 400. + const dd = clean["dataset_description.json"]; + const rawFromDoc = + dd && typeof dd === "object" && !Array.isArray(dd) ? dd.Name : undefined; + if ( + rawFromDoc !== undefined && + rawFromDoc !== null && + typeof rawFromDoc !== "string" + ) { + return res.status(400).json({ + error: 'The "Name" in dataset_description.json must be text.', + }); + } + // Spaces-only counts as no name; a repeated ?datasetName (array) is ignored. + const nameFromDoc = typeof rawFromDoc === "string" ? rawFromDoc.trim() : ""; + const nameFromQuery = + typeof req.query.datasetName === "string" + ? req.query.datasetName.trim() + : ""; + const datasetName = nameFromDoc || nameFromQuery; + if (!datasetName) { + return res.status(400).json({ error: "Please enter a dataset name." }); + } + // Matches the dataset_name VARCHAR(255) column (clear 400, not a DB error). + if (datasetName.length > 255) { + return res.status(400).json({ + error: "Dataset name is too long (255 characters max).", + }); + } + if (!nameFromDoc) { + if (!dd || typeof dd !== "object" || Array.isArray(dd)) { + clean["dataset_description.json"] = {}; + } + clean["dataset_description.json"].Name = datasetName; + } + + const internalId = req.query.internalId + ? String(req.query.internalId) + : null; + const requestedId = req.query.requestedDatasetId + ? String(req.query.requestedDatasetId).trim() + : null; + const requestedDb = req.query.requestedDb + ? String(req.query.requestedDb).trim() + : null; + const confirm = req.query.confirm === "true"; + const userId = req.user.id; + + if (requestedId) validateRequestedId(requestedId); // throws 400 + if (requestedDb) validateRequestedDb(requestedDb); // throws 400 + + // Review status of the cycle this upload lands in (returned to the client). + let reviewStatus = "draft"; + + // Keep the PostgreSQL transaction open across the CouchDB write: commit on + // success, roll back if the upload fails. Acceptable at low upload volume. + const result = await sequelize.transaction(async (t) => { + let registryId; // = internal_id (also the sandbox _id) + let submissionId; + + if (!internalId) { + // NEW logical dataset. + if (requestedId) await assertRequestedIdFree(requestedId, null, t); + registryId = crypto.randomUUID(); + await sequelize.query( + `INSERT INTO dataset_registry + (internal_id, dataset_id, requested_dataset_id, requested_db, dataset_name, owner_user_id, created_at, updated_at) + VALUES (:iid, NULL, :req, :db, :name, :uid, NOW(), NOW())`, + { + replacements: { + iid: registryId, + req: requestedId, + db: requestedDb || "public", + name: datasetName, + uid: userId, + }, + transaction: t, + } + ); + submissionId = crypto.randomUUID(); + await sequelize.query( + `INSERT INTO submissions + (submission_id, dataset_registry_id, status, json_status, json_uploaded_at, created_at, updated_at) + VALUES (:sid, :rid, 'draft', 'uploaded', NOW(), NOW(), NOW())`, + { + replacements: { sid: submissionId, rid: registryId }, + transaction: t, + } + ); + } else { + // UPDATE existing logical dataset. + registryId = internalId; + const regs = await sequelize.query( + `SELECT internal_id, owner_user_id FROM dataset_registry + WHERE internal_id = :iid FOR UPDATE`, + { + replacements: { iid: registryId }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (regs.length === 0) { + throw new HttpError(400, { error: "Unknown dataset" }); + } + if (regs[0].owner_user_id !== userId) { + throw new HttpError(403, { + error: "This dataset belongs to another user", + }); + } + + // Registry holds the canonical name. Publishing settings (preferred id, + // target db) are NOT changed by an upload — they are edited only via + // PATCH /:internalId/settings, so they're ignored here on update. + await sequelize.query( + `UPDATE dataset_registry + SET dataset_name = :name, updated_at = NOW() + WHERE internal_id = :iid`, + { + replacements: { name: datasetName, iid: registryId }, + transaction: t, + } + ); + + const subs = await sequelize.query( + `SELECT id, submission_id, status FROM submissions + WHERE dataset_registry_id = :rid + ORDER BY created_at DESC FOR UPDATE`, + { + replacements: { rid: registryId }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + + // Editable = draft / changes_requested → reuse this cycle. The review + // status is NOT changed; the user must click Submit/Resubmit to send it + // to review. + const editable = subs.find( + (s) => s.status === "draft" || s.status === "changes_requested" + ); + if (editable) { + submissionId = editable.submission_id; + reviewStatus = editable.status; + jsonTargetId = editable.id; + await sequelize.query( + `UPDATE submissions + SET json_status='uploaded', json_uploaded_at=NOW(), json_error=NULL, updated_at=NOW() + WHERE id=:id`, + { replacements: { id: editable.id }, transaction: t } + ); + } else if (subs.length === 0) { + // Defensive: registry with no cycle → start one. + submissionId = crypto.randomUUID(); + await sequelize.query( + `INSERT INTO submissions + (submission_id, dataset_registry_id, status, json_status, json_uploaded_at, created_at, updated_at) + VALUES (:sid, :rid, 'draft', 'uploaded', NOW(), NOW(), NOW())`, + { + replacements: { sid: submissionId, rid: registryId }, + transaction: t, + } + ); + } else { + const latest = subs[0]; + if (latest.status === "pending") { + // Locked while waiting for review; the user can withdraw it to + // draft first. + throw new HttpError(409, { + code: "SUBMISSION_UNDER_REVIEW", + status: "pending", + requiresConfirmation: false, + message: + "This submission is waiting for review. Withdraw it back to draft to make changes.", + }); + } + if (latest.status === "approved") { + // Still an ACTIVE workflow (awaiting promotion) — block. + throw new HttpError(409, { + code: "DATASET_ALREADY_APPROVED", + status: "approved", + requiresConfirmation: false, + message: + "This dataset has been approved and is awaiting promotion.", + }); + } + // promoted or rejected → confirmable resubmission. + if (!confirm) { + const promoted = latest.status === "promoted"; + throw new HttpError(409, { + code: promoted + ? "DATASET_ALREADY_PROMOTED" + : "PREVIOUS_SUBMISSION_REJECTED", + status: latest.status, + requiresConfirmation: true, + message: promoted + ? "This dataset is already published. Confirm to start a new version (saved as a draft)." + : "The previous submission was rejected. Confirm to start a new version (saved as a draft).", + }); + } + submissionId = crypto.randomUUID(); + await sequelize.query( + `INSERT INTO submissions + (submission_id, dataset_registry_id, status, json_status, json_uploaded_at, created_at, updated_at) + VALUES (:sid, :rid, 'draft', 'uploaded', NOW(), NOW(), NOW())`, + { + replacements: { sid: submissionId, rid: registryId }, + transaction: t, + } + ); + } + } + + // Full-replace the sandbox working copy at _id = internal_id. + // If this throws, the transaction rolls back → no rows leak. + const url = `${UPLOAD_URL}/${UPLOAD_DB}/_design/qq/_update/timestamp/${encodeURIComponent( + registryId + )}`; + await axios.put(url, clean, { headers: buildHeaders() }); + + return { internal_id: registryId, submission_id: submissionId }; + }); + + res.status(201).json({ ok: true, status: reviewStatus, ...result }); + } catch (err) { + if (err instanceof HttpError) { + return res.status(err.status).json(err.body); + } + // The transaction rolled back, so the sandbox still has the previous good + // copy. Mark the last JSON update as failed — only for an existing + // draft/changes_requested cycle (a brand-new dataset has no row to mark). + // Stored message is a fixed, safe sentence (never the raw error). + if (jsonTargetId) { + await sequelize + .query( + `UPDATE submissions + SET json_status='failed', json_error=:msg, updated_at=NOW() + WHERE id=:id`, + { + replacements: { + id: jsonTargetId, + msg: "The last JSON update could not be saved.", + }, + } + ) + .catch((e) => + console.error("Could not record JSON failure:", e.message) + ); + } + // Full details stay in the server log only. Never send err.message or the + // CouchDB response to the browser — a network error message contains the + // sandbox host/port (e.g. "connect ECONNREFUSED :7777"). + console.error( + "Upload failed:", + err.response?.status, + err.message, + "| couch:", + JSON.stringify(err.response?.data) + ); + // 502 = the upstream sandbox failed; keeps it distinct from our own 409s. + res.status(502).json({ + error: "Upload failed. Please try again later.", + }); + } +}; + +// GET /api/v1/uploads/:internalId/document — the stored sandbox doc (owner +// only). Proxied server-side so the sandbox CouchDB URL never reaches the +// browser (not even in DevTools). +const getUploadDocument = async (req, res) => { + try { + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + const url = `${UPLOAD_URL}/${UPLOAD_DB}/${encodeURIComponent( + reg.internal_id + )}`; + const { data } = await axios.get(url, { headers: buildHeaders() }); + res.json(data); + } catch (err) { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + if (err.response?.status === 404) { + return res.status(404).json({ + error: "The submitted document was not found in the sandbox.", + }); + } + console.error("Get upload document failed:", err.message); + res.status(502).json({ error: "Failed to load the submitted document." }); + } +}; + +// One row per logical dataset the caller owns (registry + its latest cycle). +const listMyUploads = async (req, res) => { + try { + const rows = await sequelize.query( + `SELECT r.internal_id, r.dataset_id, r.requested_dataset_id, r.dataset_name, + s.submission_id, s.status, + s.created_at, s.updated_at, s.promoted_db, s.promoted_at, + s.raw_zip_expected, + (SELECT o.size_bytes + FROM submission_raw_files f + JOIN raw_objects o ON o.raw_id = f.raw_id + WHERE f.submission_id = s.submission_id) AS raw_zip_size, + EXISTS (SELECT 1 FROM raw_upload_attempts a + WHERE a.submission_id = s.submission_id + AND a.status IN ('initiated','uploading','verifying')) AS raw_upload_active + FROM dataset_registry r + LEFT JOIN LATERAL ( + SELECT * FROM submissions s2 + WHERE s2.dataset_registry_id = r.internal_id + ORDER BY s2.created_at DESC LIMIT 1 + ) s ON true + WHERE r.owner_user_id = :uid + ORDER BY r.created_at DESC`, + { replacements: { uid: req.user.id }, type: sequelize.QueryTypes.SELECT } + ); + res.json(rows); + } catch (err) { + console.error("List uploads failed:", err.message); + res.status(500).json({ error: "Failed to list uploads" }); + } +}; + +const latestSubmissionId = async (internalId) => { + const s = await sequelize.query( + `SELECT submission_id FROM submissions + WHERE dataset_registry_id = :iid ORDER BY created_at DESC LIMIT 1`, + { replacements: { iid: internalId }, type: sequelize.QueryTypes.SELECT } + ); + return s[0] ? s[0].submission_id : null; +}; + +// What must be true before draft/changes_requested → pending: the JSON is +// uploaded, no raw upload is in progress, and — if this version is marked as +// including raw data — a raw object is attached. `sub` needs the +// RAW_READINESS_COLUMNS below (has_raw, raw_upload_active). +const getReadiness = (sub) => { + const problems = []; + if (sub.json_status !== "uploaded") { + problems.push("The last JSON update failed. Upload the JSON again."); + } + if (sub.raw_upload_active) { + problems.push( + "A raw data upload is still in progress. Wait for it to finish, or cancel it." + ); + } + if (sub.raw_zip_expected && !sub.has_raw) { + problems.push( + "This version is marked as including raw data, but no raw ZIP has been uploaded yet." + ); + } + return { canSubmit: problems.length === 0, problems }; +}; + +// Extra readiness columns for a `FROM submissions` query (raw ZIP state). +const RAW_READINESS_COLUMNS = ` + EXISTS (SELECT 1 FROM submission_raw_files f + WHERE f.submission_id = submissions.submission_id) AS has_raw, + EXISTS (SELECT 1 FROM raw_upload_attempts a + WHERE a.submission_id = submissions.submission_id + AND a.status IN ('initiated','uploading','verifying')) AS raw_upload_active`; + +// POST /api/v1/uploads/:internalId/submit — send (or resend) to review. +const submitForReview = async (req, res) => { + try { + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + const out = await sequelize.transaction(async (t) => { + const rows = await sequelize.query( + `SELECT id, status, json_status, raw_zip_expected, ${RAW_READINESS_COLUMNS} + FROM submissions + WHERE dataset_registry_id = :iid + ORDER BY created_at DESC LIMIT 1 FOR UPDATE`, + { + replacements: { iid: reg.internal_id }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + const sub = rows[0]; + if (!sub || !["draft", "changes_requested"].includes(sub.status)) { + throw new HttpError(409, { + code: "NOT_SUBMITTABLE", + error: + "Only a draft or a submission with requested changes can be sent for review.", + }); + } + const { canSubmit, problems } = getReadiness(sub); + if (!canSubmit) { + throw new HttpError(409, { + code: "NOT_READY", + error: problems[0], + problems, + }); + } + await sequelize.query( + `UPDATE submissions + SET status='pending', submitted_at=NOW(), updated_at=NOW() + WHERE id=:id`, + { replacements: { id: sub.id }, transaction: t } + ); + return { status: "pending" }; + }); + res.json(out); + } catch (err) { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + console.error("Submit failed:", err.message); + res.status(500).json({ error: "Failed to submit for review." }); + } +}; + +// POST /api/v1/uploads/:internalId/withdraw — pending → draft, only until the +// reviewer changes the status. A single conditional UPDATE, so it can't race a +// reviewer action: whichever lands first wins. +const withdrawSubmission = async (req, res) => { + try { + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + const sid = await latestSubmissionId(reg.internal_id); + const rows = sid + ? await sequelize.query( + `UPDATE submissions SET status='draft', updated_at=NOW() + WHERE submission_id = :sid AND status = 'pending' + RETURNING id`, + { replacements: { sid }, type: sequelize.QueryTypes.SELECT } + ) + : []; + if (rows.length === 0) { + return res.status(409).json({ + code: "CANNOT_WITHDRAW", + error: + "This submission can no longer be withdrawn — the reviewer has already acted on it.", + }); + } + res.json({ status: "draft" }); + } catch (err) { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + console.error("Withdraw failed:", err.message); + res.status(500).json({ error: "Failed to withdraw." }); + } +}; + +// PATCH /api/v1/uploads/:internalId/settings — change publishing settings +// without re-uploading. A field left out = unchanged; null or "" = reset +// (requestedDatasetId → let NeuroJSON assign; requestedDb → "public"). +const updateSettings = async (req, res) => { + try { + const body = req.body || {}; + const has = (k) => Object.prototype.hasOwnProperty.call(body, k); + const clean = (v) => + v === null || v === undefined ? null : String(v).trim() || null; + + const changeId = has("requestedDatasetId"); + const changeDb = has("requestedDb"); + if (!changeId && !changeDb) { + return res.status(400).json({ error: "Nothing to update." }); + } + const newId = changeId ? clean(body.requestedDatasetId) : null; + const newDb = changeDb ? clean(body.requestedDb) || "public" : null; + if (newId) validateRequestedId(newId); // 400 on bad format / njds + if (changeDb) validateRequestedDb(newDb); // "public" passes + + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + + // Once a dataset has a public ID, its URL (/db//) is fixed so + // shared/cited links keep working; later versions publish to the same place. + if (reg.dataset_id) { + throw new HttpError(409, { + code: "PUBLIC_ID_ASSIGNED", + error: + "This dataset is already published, so its database and ID can't be changed.", + }); + } + + const out = await sequelize.transaction(async (t) => { + // Same rule as uploads: only while the submission is editable. + const rows = await sequelize.query( + `SELECT status FROM submissions WHERE dataset_registry_id = :iid + ORDER BY created_at DESC LIMIT 1 FOR UPDATE`, + { + replacements: { iid: reg.internal_id }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if ( + !rows[0] || + !["draft", "changes_requested"].includes(rows[0].status) + ) { + throw new HttpError(409, { + code: "NOT_EDITABLE", + error: + "Settings can only be changed while the submission is a draft or has requested changes.", + }); + } + if (newId) await assertRequestedIdFree(newId, reg.internal_id, t); + + const sets = []; + const repl = { iid: reg.internal_id }; + if (changeId) { + sets.push("requested_dataset_id = :rid"); + repl.rid = newId; + } + if (changeDb) { + sets.push("requested_db = :rdb"); + repl.rdb = newDb; + } + const updated = await sequelize.query( + `UPDATE dataset_registry SET ${sets.join(", ")}, updated_at = NOW() + WHERE internal_id = :iid + RETURNING requested_dataset_id, requested_db`, + { + replacements: repl, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + return updated[0]; + }); + res.json(out); + } catch (err) { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + // Unique index backstop (two users grabbing the same ID at once). + if (err.original?.code === "23505") { + return res.status(409).json({ + code: "REQUESTED_ID_TAKEN", + error: "That preferred ID is already in use.", + }); + } + console.error("Update settings failed:", err.message); + res.status(500).json({ error: "Failed to update settings." }); + } +}; + +// GET /api/v1/uploads/:internalId — one dataset's detail (owner only). +const getUpload = async (req, res) => { + try { + const reg = await loadOwnedRegistry(String(req.params.internalId), req.user.id); + const s = await sequelize.query( + `SELECT submission_id, status, created_at, updated_at, promoted_db, promoted_at, + json_status, json_uploaded_at, json_error, submitted_at, raw_zip_expected, + reviewed_at, ${RAW_READINESS_COLUMNS} + FROM submissions WHERE dataset_registry_id = :iid + ORDER BY created_at DESC LIMIT 1`, + { replacements: { iid: reg.internal_id }, type: sequelize.QueryTypes.SELECT } + ); + const sub = s[0] || {}; + res.json({ + internal_id: reg.internal_id, + dataset_id: reg.dataset_id, + requested_dataset_id: reg.requested_dataset_id, + requested_db: reg.requested_db, + dataset_name: reg.dataset_name, + submission_id: sub.submission_id || null, + status: sub.status || null, + created_at: sub.created_at || null, + updated_at: sub.updated_at || null, + promoted_db: sub.promoted_db || null, + promoted_at: sub.promoted_at || null, + json_status: sub.json_status || null, + json_uploaded_at: sub.json_uploaded_at || null, + json_error: sub.json_error || null, + submitted_at: sub.submitted_at || null, + reviewed_at: sub.reviewed_at || null, + raw_zip_expected: !!sub.raw_zip_expected, + readiness: sub.submission_id + ? getReadiness(sub) + : { canSubmit: false, problems: ["No submission found."] }, + }); + } catch (err) { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + console.error("Get upload failed:", err.message); + res.status(500).json({ error: "Failed to load dataset" }); + } +}; + +// GET /api/v1/uploads/:internalId/comments — thread for the latest cycle. +const listComments = async (req, res) => { + try { + const reg = await loadOwnedRegistry(String(req.params.internalId), req.user.id); + const sid = await latestSubmissionId(reg.internal_id); + if (!sid) return res.json([]); + // Mask reviewer identity: the owner sees their own name, reviewers show as + // a generic "Reviewer" and their user id is not exposed. + const rows = await sequelize.query( + `SELECT c.id, + CASE WHEN c.user_id = :owner THEN u.username ELSE 'Reviewer' END AS username, + c.message, c.created_at, + (c.user_id = :owner) AS is_owner + FROM submission_comments c + JOIN users u ON u.id = c.user_id + WHERE c.submission_id = :sid + ORDER BY c.created_at ASC`, + { + replacements: { sid, owner: reg.owner_user_id }, + type: sequelize.QueryTypes.SELECT, + } + ); + res.json(rows); + } catch (err) { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + console.error("List comments failed:", err.message); + res.status(500).json({ error: "Failed to load comments" }); + } +}; + +// POST /api/v1/uploads/:internalId/comments — add a comment (owner only for now). +const postComment = async (req, res) => { + try { + const reg = await loadOwnedRegistry(String(req.params.internalId), req.user.id); + const message = String((req.body && req.body.message) || "").trim(); + if (!message) throw new HttpError(400, { error: "Message cannot be empty." }); + if (message.length > 5000) { + throw new HttpError(400, { + error: "Message is too long (5000 characters max).", + }); + } + const sid = await latestSubmissionId(reg.internal_id); + if (!sid) throw new HttpError(400, { error: "No submission to comment on." }); + const ins = await sequelize.query( + `INSERT INTO submission_comments (submission_id, user_id, message, created_at) + VALUES (:sid, :uid, :msg, NOW()) RETURNING id, created_at`, + { + replacements: { sid, uid: req.user.id, msg: message }, + type: sequelize.QueryTypes.SELECT, + } + ); + const isOwner = reg.owner_user_id === req.user.id; + res.status(201).json({ + id: ins[0].id, + username: isOwner ? req.user.username : "Reviewer", + message, + created_at: ins[0].created_at, + is_owner: isOwner, + }); + } catch (err) { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + console.error("Post comment failed:", err.message); + res.status(500).json({ error: "Failed to post comment" }); + } +}; + +module.exports = { + createUpload, + listMyUploads, + getUpload, + getUploadDocument, + submitForReview, + withdrawSubmission, + updateSettings, + listComments, + postComment, +}; diff --git a/backend/src/jobs/reconcileRawUploads.js b/backend/src/jobs/reconcileRawUploads.js new file mode 100644 index 0000000..db0f425 --- /dev/null +++ b/backend/src/jobs/reconcileRawUploads.js @@ -0,0 +1,180 @@ +// Reconciliation: REN asks the Zodiac storage API about raw uploads that +// aren't finished and finishes them. Required in dev (Zodiac can't reach the +// developer's machine, so pushes are off); a safety net in prod. Decisions +// reuse processStorageEvent, so pull and push can never decide differently. +// Assumes ONE backend process (add a Postgres advisory lock if that changes). +const { sequelize } = require("../config/database"); +const { + processStorageEvent, +} = require("../controllers/storageEvents.controller"); +const storage = require("../lib/storageClient"); + +const INTERVAL_MS = Number(process.env.RAW_RECONCILE_INTERVAL_MS ?? 600000); // 0 = off +const MIN_AGE_MS = Number(process.env.RAW_RECONCILE_MIN_AGE_MS ?? 1800000); // let pushes happen first +const EXPIRE_MS = Number(process.env.RAW_INACTIVE_EXPIRE_DAYS ?? 7) * 86400000; +const BATCH = 50; +const ACTIVE = ["initiated", "uploading", "verifying"]; + +const q = (sql, replacements) => + sequelize.query(sql, { replacements, type: sequelize.QueryTypes.SELECT }); +const run = (sql, replacements) => sequelize.query(sql, { replacements }); + +const touch = (id) => + run( + `UPDATE raw_upload_attempts SET last_checked_at = NOW() WHERE upload_id = :id`, + { id } + ); +const expire = (id, msg) => + run( + `UPDATE raw_upload_attempts + SET status = 'expired', error = :msg, completed_at = NOW(), updated_at = NOW() + WHERE upload_id = :id AND status IN (:active)`, + { id, msg, active: ACTIVE } + ); +const idleTooLong = (since) => + Date.now() - new Date(since).getTime() > EXPIRE_MS; + +// One unfinished attempt: look at Zodiac, then take the matching step. +const reconcileActive = async (a) => { + const id = a.upload_id; + const st = await storage.getUpload(id); + switch (st.phase) { + case "none": + if (idleTooLong(a.updated_at)) { + await expire(id, "The upload was never started."); + } + break; + case "incoming": + if (idleTooLong(st.lastActivityAt)) { + await expire(id, "The upload stopped and expired."); + await storage.deleteUpload(id); + } else { + await run( + `UPDATE raw_upload_attempts SET status = 'uploading', updated_at = NOW() + WHERE upload_id = :id AND status = 'initiated'`, + { id } + ); + } + break; + case "awaiting_decision": { + const reply = await processStorageEvent(id, { + event: "verified", + size: st.size, + sha256: st.sha256, + }); + await storage.sendDecision(id, reply); + break; + } + case "committed": { + const reply = await processStorageEvent(id, { + event: "committed", + size: st.size, + sha256: st.sha256, + }); + await storage.sendAck(id, reply); + break; + } + case "rejecting": + await processStorageEvent(id, { event: "failed", reason: st.reason }); + await storage.sendAck(id); + break; + case "failed": // parked on Zodiac for an admin + await processStorageEvent(id, { + event: "failed", + reason: "storage_error", + }); + break; + default: + // verifying / committing / discarding / done: Zodiac is busy (or + // finished a discard) — check again next run. + break; + } + await touch(id); + return st.phase; +}; + +// A finished-in-REN attempt (cancelled/failed/expired): make sure Zodiac +// drops whatever it still holds. Never touches committed objects REN kept. +const cleanUpFinished = async (a) => { + const id = a.upload_id; + const st = await storage.getUpload(id); + if (st.phase === "incoming") { + await storage.deleteUpload(id); + } else if (st.phase === "awaiting_decision") { + await storage.sendDecision(id, { decision: "discard", reason: a.status }); + } else if (st.phase === "committed") { + await storage.sendAck(id, { action: "discard", reason: a.status }); + } else if (st.phase === "rejecting") { + await storage.sendAck(id); + } + await touch(id); // cleaned up → skipped next time + return st.phase; +}; + +let running = false; + +// One full pass. Returns a short summary (used by the one-shot script too). +const reconcileOnce = async () => { + if (running) return { skipped: true }; + running = true; + const summary = { active: 0, cleaned: 0, errors: 0 }; + try { + const active = await q( + `SELECT upload_id, status, updated_at FROM raw_upload_attempts + WHERE status IN (:active) + AND updated_at < NOW() - (:minAge * INTERVAL '1 millisecond') + ORDER BY updated_at LIMIT ${BATCH}`, + { active: ACTIVE, minAge: MIN_AGE_MS } + ); + for (const a of active) { + try { + const phase = await reconcileActive(a); + summary.active++; + console.log(`reconcile ${a.upload_id}: Zodiac says ${phase}`); + } catch (e) { + summary.errors++; + console.warn(`reconcile ${a.upload_id} failed (will retry):`, e.message); + } + } + const finished = await q( + `SELECT upload_id, status FROM raw_upload_attempts + WHERE status IN ('cancelled','failed','expired') + AND (last_checked_at IS NULL OR last_checked_at < completed_at) + ORDER BY completed_at LIMIT ${BATCH}` + ); + for (const a of finished) { + try { + const phase = await cleanUpFinished(a); + summary.cleaned++; + console.log(`cleanup ${a.upload_id} (${a.status}): Zodiac had ${phase}`); + } catch (e) { + summary.errors++; + console.warn(`cleanup ${a.upload_id} failed (will retry):`, e.message); + } + } + } finally { + running = false; + } + return summary; +}; + +const startRawReconciler = () => { + if (!INTERVAL_MS) { + return console.log("raw reconciler: off (RAW_RECONCILE_INTERVAL_MS=0)"); + } + if (!storage.configured()) { + return console.log( + "raw reconciler: off (no STORAGE_INTERNAL_URL / STORAGE_PUBLIC_URL)" + ); + } + console.log( + `raw reconciler: every ${INTERVAL_MS / 1000}s (min age ${MIN_AGE_MS / 1000}s)` + ); + setInterval( + () => + reconcileOnce().catch((e) => console.error("raw reconciler:", e.message)), + INTERVAL_MS + ); +}; + +module.exports = { reconcileOnce, startRawReconciler }; diff --git a/backend/src/lib/storageClient.js b/backend/src/lib/storageClient.js new file mode 100644 index 0000000..0c8cc05 --- /dev/null +++ b/backend/src/lib/storageClient.js @@ -0,0 +1,55 @@ +// Signed calls from REN to the Zodiac storage API's /internal endpoints +// (HMAC, same scheme as Zodiac → REN). Throws on any non-2xx so callers can +// simply try again on the next run. +const { signRequest } = require("./storageSignature"); + +const base = () => + ( + process.env.STORAGE_INTERNAL_URL || + process.env.STORAGE_PUBLIC_URL || + "" + ).replace(/\/+$/, ""); + +const call = async (method, sub, body) => { + const url = new URL(`${base()}/internal${sub}`); + const json = body === undefined ? "" : JSON.stringify(body); + const headers = { + ...(json ? { "Content-Type": "application/json" } : {}), + // Zodiac verifies against its req.originalUrl = this exact path + query. + ...signRequest({ method, path: url.pathname + url.search, body: json }), + }; + const res = await fetch(url, { + method, + headers, + body: json || undefined, + signal: AbortSignal.timeout(30_000), + }); + const text = await res.text(); + if (!res.ok) { + throw new Error( + `storage ${method} ${sub} → HTTP ${res.status}: ${text.slice(0, 200)}` + ); + } + return JSON.parse(text); +}; + +module.exports = { + configured: () => Boolean(base()), + getUpload: (id) => call("GET", `/uploads/${id}`), + // Forward REN's answer exactly as it would be returned to a push. + sendDecision: (id, reply) => + call("POST", `/uploads/${id}/decision`, { + decision: reply.decision, + rawId: reply.rawId, + storageKey: reply.storageKey, + reason: reply.reason, + }), + sendAck: (id, reply = {}) => + call( + "POST", + `/uploads/${id}/ack`, + reply.action ? { action: reply.action, reason: reply.reason } : {} + ), + deleteUpload: (id) => call("DELETE", `/uploads/${id}`), + status: () => call("GET", "/status"), +}; diff --git a/backend/src/lib/storageSignature.js b/backend/src/lib/storageSignature.js new file mode 100644 index 0000000..9d4ba75 --- /dev/null +++ b/backend/src/lib/storageSignature.js @@ -0,0 +1,70 @@ +// HMAC-SHA256 signatures for server-to-server calls between REN and the +// Zodiac storage API (both directions). Signs timestamp + method + path(+query) +// + SHA-256 of the raw body. Secrets are a kid -> secret map so they can rotate. +const crypto = require("crypto"); + +const MAX_SKEW_SECONDS = 300; +const H_KID = "x-nj-key-id"; +const H_TS = "x-nj-timestamp"; +const H_SIG = "x-nj-signature"; + +const loadKeys = () => { + try { + return JSON.parse(process.env.STORAGE_HMAC_KEYS || "{}"); + } catch { + throw new Error("STORAGE_HMAC_KEYS is not valid JSON."); + } +}; + +const sha256hex = (buf) => + crypto.createHash("sha256").update(buf || "").digest("hex"); +const canonical = (ts, method, path, body) => + [ts, String(method).toUpperCase(), path, sha256hex(body)].join("\n"); + +// Headers to attach to an outgoing request. `body` = the exact bytes/string sent. +const signRequest = ({ method, path, body = "" }) => { + const kid = process.env.STORAGE_HMAC_ACTIVE_KID; + const secret = loadKeys()[kid]; + if (!kid || !secret) throw new Error("Storage HMAC key not configured."); + const ts = String(Math.floor(Date.now() / 1000)); + const sig = crypto + .createHmac("sha256", secret) + .update(canonical(ts, method, path, body)) + .digest("base64url"); + return { [H_KID]: kid, [H_TS]: ts, [H_SIG]: sig }; +}; + +// Check an incoming request. `path` must include the query string exactly as +// sent; `rawBody` must be the unparsed bytes. Returns { ok, reason }. +const verifyRequest = ({ method, path, headers, rawBody }) => { + const kid = headers[H_KID]; + const ts = headers[H_TS]; + const sig = headers[H_SIG]; + if (!kid || !ts || !sig) { + return { ok: false, reason: "missing signature headers" }; + } + const secret = loadKeys()[kid]; + if (!secret) return { ok: false, reason: "unknown key id" }; + const now = Math.floor(Date.now() / 1000); + if (!/^\d+$/.test(ts) || Math.abs(now - Number(ts)) > MAX_SKEW_SECONDS) { + return { ok: false, reason: "timestamp out of range" }; + } + const expected = crypto + .createHmac("sha256", secret) + .update(canonical(ts, method, path, rawBody)) + .digest(); + const given = Buffer.from(String(sig), "base64url"); + if ( + given.length !== expected.length || + !crypto.timingSafeEqual(given, expected) + ) { + return { ok: false, reason: "bad signature" }; + } + return { ok: true, kid }; +}; + +module.exports = { + signRequest, + verifyRequest, + HEADER_NAMES: { H_KID, H_TS, H_SIG }, +}; diff --git a/backend/src/lib/uploadAccess.js b/backend/src/lib/uploadAccess.js new file mode 100644 index 0000000..6ce8c46 --- /dev/null +++ b/backend/src/lib/uploadAccess.js @@ -0,0 +1,34 @@ +// Shared by uploads.controller.js and rawUploads.controller.js. +const { sequelize } = require("../config/database"); + +// Errors thrown inside handlers/transactions to signal a specific HTTP response. +class HttpError extends Error { + constructor(status, body) { + super(typeof body === "string" ? body : body.error || "error"); + this.status = status; + this.body = typeof body === "string" ? { error: body } : body; + } +} + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +// Load a registry row the caller owns, or throw (404 unknown, 403 not owner). +// Guards the UUID format first so a bad param can't crash the SQL uuid cast. +const loadOwnedRegistry = async (internalId, userId) => { + if (!UUID_RE.test(internalId)) { + throw new HttpError(404, { error: "Dataset not found" }); + } + const rows = await sequelize.query( + `SELECT internal_id, dataset_id, requested_dataset_id, requested_db, dataset_name, owner_user_id + FROM dataset_registry WHERE internal_id = :iid`, + { replacements: { iid: internalId }, type: sequelize.QueryTypes.SELECT } + ); + if (rows.length === 0) throw new HttpError(404, { error: "Dataset not found" }); + if (rows[0].owner_user_id !== userId) { + throw new HttpError(403, { error: "This dataset belongs to another user" }); + } + return rows[0]; +}; + +module.exports = { HttpError, UUID_RE, loadOwnedRegistry }; diff --git a/backend/src/lib/uploadTokens.js b/backend/src/lib/uploadTokens.js new file mode 100644 index 0000000..55227df --- /dev/null +++ b/backend/src/lib/uploadTokens.js @@ -0,0 +1,121 @@ +// Short-lived EdDSA (Ed25519) JWTs that let the browser talk to the Zodiac +// storage API. REN signs with the private key; Zodiac verifies with a +// kid -> public-key map (so keys can rotate). Built on Node's crypto (no jose: +// jose 6 is ESM-only and this backend is CommonJS). +const crypto = require("crypto"); + +const ISSUER = "neurojson-api"; +const AUDIENCE = "zodiac-storage"; +const DEFAULT_TTL_SECONDS = 15 * 60; +const CLOCK_LEEWAY_SECONDS = 30; + +const b64url = (v) => + Buffer.from(typeof v === "string" ? v : JSON.stringify(v)).toString( + "base64url" + ); + +let signer; // cached { key, kid } +const getSigner = () => { + if (signer) return signer; + const pem = process.env.UPLOAD_TOKEN_PRIVATE_KEY; + const kid = process.env.UPLOAD_TOKEN_KID; + if (!pem || !kid) { + throw new Error( + "Upload token key not configured (UPLOAD_TOKEN_PRIVATE_KEY / UPLOAD_TOKEN_KID)." + ); + } + const key = crypto.createPrivateKey(pem); + if (key.asymmetricKeyType !== "ed25519") { + throw new Error("UPLOAD_TOKEN_PRIVATE_KEY must be an Ed25519 key."); + } + signer = { key, kid }; + return signer; +}; + +// Sign a token of a given purpose (typ) with extra claims. +const signToken = (typ, claims, ttlSeconds = DEFAULT_TTL_SECONDS) => { + const { key, kid } = getSigner(); + const iat = Math.floor(Date.now() / 1000); + const header = { alg: "EdDSA", typ: "JWT", kid }; + const payload = { + iss: ISSUER, + aud: AUDIENCE, + typ, + iat, + exp: iat + ttlSeconds, + jti: crypto.randomUUID(), + ...claims, + }; + const input = `${b64url(header)}.${b64url(payload)}`; + const sig = crypto.sign(null, Buffer.from(input), key).toString("base64url"); + return { + token: `${input}.${sig}`, + expiresAt: new Date(payload.exp * 1000).toISOString(), + }; +}; + +// Token for all tus requests of ONE upload. `len` = exact Upload-Length. +const signUploadToken = ({ + userId, + uploadId, + internalId, + submissionId, + expectedSize, +}) => + signToken("tus-upload", { + sub: String(userId), + uid: uploadId, + iid: internalId, + sid: submissionId, + len: expectedSize, + }); + +// Strict verification (used by Zodiac; also by REN's self-test). +// publicKeys: { [kid]: pem | KeyObject }. Throws on any problem. +const verifyToken = (token, publicKeys, expectedTyp) => { + const parts = String(token || "").split("."); + if (parts.length !== 3) throw new Error("malformed token"); + const [h, p, s] = parts; + const header = JSON.parse(Buffer.from(h, "base64url").toString()); + if (header.alg !== "EdDSA") throw new Error("bad alg"); // never trust other algs + const keyMaterial = publicKeys[header.kid]; + if (!keyMaterial) throw new Error("unknown kid"); + const key = + typeof keyMaterial === "string" + ? crypto.createPublicKey(keyMaterial) + : keyMaterial; + if (key.asymmetricKeyType !== "ed25519") throw new Error("bad key type"); + const ok = crypto.verify( + null, + Buffer.from(`${h}.${p}`), + key, + Buffer.from(s, "base64url") + ); + if (!ok) throw new Error("bad signature"); + const claims = JSON.parse(Buffer.from(p, "base64url").toString()); + const now = Math.floor(Date.now() / 1000); + if (claims.iss !== ISSUER || claims.aud !== AUDIENCE) { + throw new Error("bad issuer/audience"); + } + if (claims.typ !== expectedTyp) throw new Error("wrong token type"); + if ( + typeof claims.exp !== "number" || + claims.exp + CLOCK_LEEWAY_SECONDS < now + ) { + throw new Error("expired"); + } + if ( + typeof claims.iat !== "number" || + claims.iat - CLOCK_LEEWAY_SECONDS > now + ) { + throw new Error("issued in the future"); + } + return claims; +}; + +module.exports = { + signToken, + signUploadToken, + verifyToken, + DEFAULT_TTL_SECONDS, +}; diff --git a/backend/src/middleware/verifyStorageSignature.js b/backend/src/middleware/verifyStorageSignature.js new file mode 100644 index 0000000..28e4d1e --- /dev/null +++ b/backend/src/middleware/verifyStorageSignature.js @@ -0,0 +1,25 @@ +// Only the Zodiac storage API may call /api/v1/internal/*: every request must +// carry a valid HMAC signature over method + path + raw body (see +// src/lib/storageSignature.js). Needs req.rawBody (set by the internal router). +const { verifyRequest } = require("../lib/storageSignature"); + +const verifyStorageSignature = (req, res, next) => { + const result = verifyRequest({ + method: req.method, + path: req.originalUrl, // includes the query string exactly as sent + headers: req.headers, + rawBody: req.rawBody || "", + }); + if (!result.ok) { + console.warn( + "Rejected internal storage call:", + result.reason, + req.method, + req.originalUrl + ); + return res.status(401).json({ error: "Invalid service signature." }); + } + next(); +}; + +module.exports = { verifyStorageSignature }; diff --git a/backend/src/routes/internal.routes.js b/backend/src/routes/internal.routes.js new file mode 100644 index 0000000..371ee67 --- /dev/null +++ b/backend/src/routes/internal.routes.js @@ -0,0 +1,26 @@ +// Server-to-server routes called by the Zodiac storage API (never the browser). +// Mounted in server.js BEFORE the global express.json(), so this router can +// keep the raw body for the HMAC check. +const express = require("express"); +const { + verifyStorageSignature, +} = require("../middleware/verifyStorageSignature"); +const { + handleUploadEvent, +} = require("../controllers/storageEvents.controller"); + +const router = express.Router(); + +router.use( + express.json({ + limit: "1mb", + verify: (req, res, buf) => { + req.rawBody = buf; + }, + }) +); +router.use(verifyStorageSignature); + +router.post("/storage/uploads/:uploadId/events", handleUploadEvent); + +module.exports = router; diff --git a/backend/src/routes/uploads.routes.js b/backend/src/routes/uploads.routes.js new file mode 100644 index 0000000..a59ab39 --- /dev/null +++ b/backend/src/routes/uploads.routes.js @@ -0,0 +1,74 @@ +const express = require("express"); +const rateLimit = require("express-rate-limit"); +const { requireAuth } = require("../middleware/auth.middleware"); +const { + createUpload, + listMyUploads, + getUpload, + getUploadDocument, + submitForReview, + withdrawSubmission, + updateSettings, + listComments, + postComment, +} = require("../controllers/uploads.controller"); +const { + startRawUpload, + refreshRawUploadToken, + getRawStatus, + cancelRawUpload, + setRawExpected, +} = require("../controllers/rawUploads.controller"); + +const router = express.Router(); + +// Per-USER daily cap so one account can't flood the server. requireAuth runs +// first, so req.user is set and we key by user id (falls back to IP). The limit +// is configurable via UPLOAD_RATE_LIMIT_MAX (default 20) — no code change to tune. +const uploadRateLimit = rateLimit({ + windowMs: 24 * 60 * 60 * 1000, // 24 hours + max: parseInt(process.env.UPLOAD_RATE_LIMIT_MAX || "20", 10), + standardHeaders: true, + legacyHeaders: false, + // requireAuth runs first, so req.user is always set — key purely by user id. + // (Avoid referencing req.ip: express-rate-limit v7 statically flags that as + // an IPv6-unsafe key generator.) + keyGenerator: (req) => `user:${req.user.id}`, + message: { + error: "Daily upload limit reached. Please try again tomorrow.", + }, +}); + +// POST /api/v1/uploads — upload a JSON document (logged-in users only). +router.post("/", requireAuth, uploadRateLimit, createUpload); + +// GET /api/v1/uploads — the caller's own submissions (dashboard Uploads tab). +router.get("/", requireAuth, listMyUploads); + +// One dataset's detail + its review conversation (owner only). +router.get("/:internalId", requireAuth, getUpload); +router.get("/:internalId/document", requireAuth, getUploadDocument); + +// Review workflow actions (owner only). +router.post("/:internalId/submit", requireAuth, submitForReview); +router.post("/:internalId/withdraw", requireAuth, withdrawSubmission); + +// Publishing settings (target db + preferred id), editable without re-uploading. +router.patch("/:internalId/settings", requireAuth, updateSettings); +router.get("/:internalId/comments", requireAuth, listComments); +router.post("/:internalId/comments", requireAuth, postComment); + +// Raw ZIP upload sessions (owner only). The file itself goes straight to +// Zodiac; REN only authorizes and tracks state. Starting shares the daily +// upload limit; token refreshes aren't limited (long uploads need several). +router.post("/:internalId/raw", requireAuth, uploadRateLimit, startRawUpload); +router.get("/:internalId/raw", requireAuth, getRawStatus); +router.patch("/:internalId/raw/expected", requireAuth, setRawExpected); +router.post( + "/:internalId/raw/:uploadId/token", + requireAuth, + refreshRawUploadToken +); +router.delete("/:internalId/raw/:uploadId", requireAuth, cancelRawUpload); + +module.exports = router; diff --git a/backend/src/server.js b/backend/src/server.js index d0d043d..63c376b 100644 --- a/backend/src/server.js +++ b/backend/src/server.js @@ -15,6 +15,9 @@ const collectionRoutes = require("./routes/collection.route"); const projectRoutes = require("./routes/projects.routes"); const ollamaRoutes = require("./routes/ollama.routes"); const ollamaPublicRoutes = require("./routes/ollama.public.routes"); +const uploadsRoutes = require("./routes/uploads.routes"); +const internalRoutes = require("./routes/internal.routes"); +const { startRawReconciler } = require("./jobs/reconcileRawUploads"); const app = express(); const PORT = process.env.PORT || 5000; @@ -35,6 +38,10 @@ app.use( }) ); +// Zodiac → REN service calls (HMAC-signed). Mounted BEFORE express.json() so +// the internal router can read the raw body; no cookies/restoreUser needed. +app.use("/api/v1/internal", internalRoutes); + app.use(express.json({ limit: "50mb" })); app.use(express.urlencoded({ limit: "50mb", extended: true })); app.use(cookieParser()); // parse cookies @@ -53,6 +60,7 @@ app.use("/api/v1/collections", collectionRoutes); app.use("/api/v1/projects", projectRoutes); app.use("/api/v1/ollama", ollamaRoutes); app.use("/api/v1/ollama-public", ollamaPublicRoutes); +app.use("/api/v1/uploads", uploadsRoutes); // health check endpoint app.get("/api/health", async (req, res) => { @@ -99,6 +107,8 @@ const startServer = async () => { console.log(`🚀 Server is running on port ${PORT}`); console.log(`📡 API available at http://localhost:${PORT}/api`); console.log(`🌍 Environment: ${process.env.NODE_ENV || "development"}`); + // Finish raw uploads by asking the storage API (needed in pull mode). + startRawReconciler(); }); } catch (error) { console.error("❌ Failed to start server:", error); diff --git a/package.json b/package.json index 937f15d..8c837f6 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "stats-js": "^1.0.1", "stats.js": "0.17.0", "three": "0.145.0", + "tus-js-client": "^4.3.1", "typescript": "^5.1.6", "uplot": "1.6.32", "web-vitals": "^2.1.0", diff --git a/src/components/CopyButton.tsx b/src/components/CopyButton.tsx new file mode 100644 index 0000000..9f403d1 --- /dev/null +++ b/src/components/CopyButton.tsx @@ -0,0 +1,43 @@ +import CheckIcon from "@mui/icons-material/Check"; +import ContentCopyIcon from "@mui/icons-material/ContentCopy"; +import { IconButton, Tooltip } from "@mui/material"; +import React, { useState } from "react"; + +interface CopyButtonProps { + value: string; + label?: string; +} + +// Small inline "copy to clipboard" icon button with a transient "Copied!" tick. +const CopyButton: React.FC = ({ value, label = "Copy" }) => { + const [copied, setCopied] = useState(false); + + const copy = async () => { + try { + await navigator.clipboard.writeText(value); + setCopied(true); + setTimeout(() => setCopied(false), 1500); + } catch { + // Clipboard API unavailable (e.g. non-HTTPS) — silently ignore. + } + }; + + return ( + + + {copied ? ( + + ) : ( + + )} + + + ); +}; + +export default CopyButton; diff --git a/src/components/Routes.tsx b/src/components/Routes.tsx index 10b15e9..6602c22 100644 --- a/src/components/Routes.tsx +++ b/src/components/Routes.tsx @@ -13,6 +13,8 @@ import Home from "pages/Home"; import ResendVerification from "pages/ResendVerification"; import SearchPage from "pages/SearchPage"; import UpdatedDatasetDetailPage from "pages/UpdatedDatasetDetailPage"; +import UploadDetailPage from "pages/UploadDetailPage"; +import UploadPage from "pages/UploadPage"; import NewDatasetPage from "pages/UpdatedDatasetPage"; import VerifyEmail from "pages/VerifyEmail"; import React from "react"; @@ -70,6 +72,8 @@ const Routes = () => ( /> } /> } /> + } /> + } /> diff --git a/src/components/Upload/PublishSettingsDialog.tsx b/src/components/Upload/PublishSettingsDialog.tsx new file mode 100644 index 0000000..feeb02f --- /dev/null +++ b/src/components/Upload/PublishSettingsDialog.tsx @@ -0,0 +1,188 @@ +import { + Alert, + Box, + Button, + CircularProgress, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + FormControlLabel, + Radio, + RadioGroup, + TextField, + Typography, +} from "@mui/material"; +import { Colors } from "design/theme"; +import React, { useEffect, useState } from "react"; +import { UploadService } from "services/upload.service"; +import { + getRequestedDbError, + getRequestedIdError, + useExistingDbNames, +} from "utils/uploadValidation"; + +interface PublishSettingsDialogProps { + open: boolean; + internalId: string; + currentDb: string | null; // "public" when unset + currentId: string | null; // null = NeuroJSON assigns + onClose: () => void; + onSaved: () => void; +} + +const radioSx = { "&.Mui-checked": { color: Colors.purple } }; + +// Edit the target database + preferred dataset id without re-uploading. +const PublishSettingsDialog: React.FC = ({ + open, + internalId, + currentDb, + currentId, + onClose, + onSaved, +}) => { + const existingDbs = useExistingDbNames(); + const [db, setDb] = useState(""); + const [mode, setMode] = useState<"auto" | "custom">("auto"); + const [customId, setCustomId] = useState(""); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + + // Reset the form to the saved values each time the dialog opens. + useEffect(() => { + if (!open) return; + setDb(currentDb && currentDb !== "public" ? currentDb : ""); + setMode(currentId ? "custom" : "auto"); + setCustomId(currentId || ""); + setError(null); + }, [open, currentDb, currentId]); + + const dbError = getRequestedDbError(db, existingDbs); + const idError = + mode === "custom" + ? customId.trim() + ? getRequestedIdError(customId) + : "Enter a preferred ID, or choose “Let NeuroJSON assign one”." + : ""; + + // Warn that the old preferred id becomes available to others. + const releasesId = + !!currentId && (mode === "auto" || customId.trim() !== currentId); + + const save = async () => { + setSaving(true); + setError(null); + try { + await UploadService.updateSettings(internalId, { + requestedDb: db.trim() || null, // null → "public" + requestedDatasetId: mode === "auto" ? null : customId.trim(), + }); + onSaved(); + } catch (e: any) { + setError(e.message || "Failed to save settings."); + } finally { + setSaving(false); + } + }; + + return ( + + Publishing settings + + setDb(e.target.value)} + size="small" + fullWidth + error={!!dbError} + helperText={ + dbError || + "Becomes part of the public URL after review (e.g. /db/smith-lab/…). Leave blank for public." + } + sx={{ mt: 1 }} + /> + + + Dataset ID + + setMode(e.target.value as "auto" | "custom")} + > + } + label={ + <> + Let NeuroJSON assign one (njds######) + + } + /> + } + label="Use my preferred ID" + /> + + {mode === "custom" && ( + setCustomId(e.target.value)} + size="small" + fullWidth + error={!!idError} + helperText={ + idError || + "Use only lowercase letters and numbers; join words with - or _." + } + sx={{ mt: 0.5, ml: 4, width: "calc(100% - 32px)" }} + /> + )} + + {releasesId && ( + + Your preferred ID {currentId} will be released, and + other users can then use it. + + )} + {error && ( + + {error} + + )} + + + + + + + ); +}; + +export default PublishSettingsDialog; diff --git a/src/components/Upload/RawDataUploader.tsx b/src/components/Upload/RawDataUploader.tsx new file mode 100644 index 0000000..50f1bd6 --- /dev/null +++ b/src/components/Upload/RawDataUploader.tsx @@ -0,0 +1,614 @@ +import { + Alert, + Box, + Button, + Checkbox, + CircularProgress, + Dialog, + DialogActions, + DialogContent, + DialogContentText, + DialogTitle, + FormControlLabel, + LinearProgress, + Paper, + Typography, +} from "@mui/material"; +import CopyButton from "components/CopyButton"; +import { Colors } from "design/theme"; +import React, { useCallback, useEffect, useRef, useState } from "react"; +import { + RawExtractionStatus, + RawStatus, + RawUploadError, + RawUploadService, +} from "services/rawUpload.service"; +import { formatBytes } from "utils/formatBytes"; +import { RawUploader, rawUploadSupported } from "utils/rawUploader"; + +// "Raw data (ZIP)" section of the upload detail page: shows the version's +// current ZIP and uploads a new one straight to the storage API (tus). + +const POLL_MS = 3000; + +type Phase = "idle" | "starting" | "uploading" | "paused" | "verifying"; + +const EXTRACTION_LABEL: Record = { + not_extracted: "Not extracted yet", + extracting: "Extracting…", + extracted: "Extracted", + failed: "Extraction failed", +}; + +const formatEta = (seconds: number) => { + if (!Number.isFinite(seconds) || seconds <= 0) return ""; + if (seconds < 60) return `about ${Math.ceil(seconds)} s left`; + if (seconds < 3600) return `about ${Math.ceil(seconds / 60)} min left`; + const h = Math.floor(seconds / 3600); + return `about ${h} h ${Math.ceil((seconds - h * 3600) / 60)} min left`; +}; + +const purpleButton = { + backgroundColor: Colors.purple, + "&:hover": { backgroundColor: Colors.secondaryPurple }, +}; +const outlinedPurple = { color: Colors.purple, borderColor: Colors.purple }; + +interface RawDataUploaderProps { + internalId: string; + status?: string | null; // submission status from the page; reload when it changes + onChanged?: () => void; // parent reloads the dataset (readiness etc.) +} + +const RawDataUploader: React.FC = ({ + internalId, + status: submissionStatus, + onChanged, +}) => { + const [status, setStatus] = useState(null); + const [loadError, setLoadError] = useState(null); + const [phase, setPhase] = useState("idle"); + const [progress, setProgress] = useState<{ + sent: number; + total: number; + speed: number; + } | null>(null); + const [error, setError] = useState<{ + message: string; + canResume: boolean; + } | null>(null); + const [conflict, setConflict] = useState<{ + uploadId: string; + filename: string; + size: number; + } | null>(null); + const [confirmCancel, setConfirmCancel] = useState(false); + const [busy, setBusy] = useState(false); + const [success, setSuccess] = useState(null); + const uploaderRef = useRef(null); + const verifyingIdRef = useRef(null); + const speedRef = useRef<{ t: number; bytes: number; speed: number } | null>( + null + ); + const fileInput = useRef(null); + + const load = useCallback(async () => { + try { + const s = await RawUploadService.getStatus(internalId); + setStatus(s); + setLoadError(null); + return s; + } catch (e: any) { + setLoadError(e.message || "Could not load raw data status."); + return null; + } + }, [internalId]); + + useEffect(() => { + load(); + }, [load, submissionStatus]); // e.g. Submit / Withdraw changes "editable" + + // Reloaded while the server was already verifying → keep waiting for it. + useEffect(() => { + if (phase === "idle" && status?.activeUpload?.status === "verifying") { + verifyingIdRef.current = status.activeUpload.uploadId; + setPhase("verifying"); + } + }, [status, phase]); + + // Verifying: poll until the new ZIP is attached, or the upload ended + // without it (failed / cancelled / expired). raw_id = upload_id on the + // server, so "attached" means current.rawId === the upload id. + useEffect(() => { + if (phase !== "verifying") return; + const id = verifyingIdRef.current; + const timer = setInterval(async () => { + const s = await load(); + if (!s || !id) return; + const attached = s.current?.rawId === id; + const stillActive = s.activeUpload?.uploadId === id; + if (!attached && stillActive) return; + clearInterval(timer); + uploaderRef.current = null; + setProgress(null); + setPhase("idle"); + if (attached && s.current) { + setSuccess( + `${s.current.filename || "The ZIP"} (${formatBytes( + s.current.size + )}) is verified and attached to this version.` + ); + } else if (s.lastUpload?.uploadId === id) { + setError({ + message: s.lastUpload.error || "The upload could not be completed.", + canResume: false, + }); + } + onChanged?.(); + }, POLL_MS); + return () => clearInterval(timer); + }, [phase, load, onChanged]); + + // Closing the tab mid-upload: ask first (the browser shows its own text). + useEffect(() => { + if (phase !== "uploading" && phase !== "starting") return; + const warn = (e: BeforeUnloadEvent) => { + e.preventDefault(); + e.returnValue = ""; + }; + window.addEventListener("beforeunload", warn); + return () => window.removeEventListener("beforeunload", warn); + }, [phase]); + + // Leaving this page inside the app: stop sending (resumable later). + useEffect( + () => () => { + uploaderRef.current?.pause(); + }, + [] + ); + + const trackSpeed = (sent: number) => { + const now = performance.now(); + const prev = speedRef.current; + if (!prev) { + speedRef.current = { t: now, bytes: sent, speed: 0 }; + return 0; + } + if (now - prev.t < 500) return prev.speed; + const instant = (sent - prev.bytes) / ((now - prev.t) / 1000); + const speed = prev.speed ? 0.3 * instant + 0.7 * prev.speed : instant; + speedRef.current = { t: now, bytes: sent, speed }; + return speed; + }; + + const startUpload = async (file: File) => { + setSuccess(null); + setError(null); + setConflict(null); + if (!/\.zip$/i.test(file.name)) { + return setError({ + message: "Please choose a .zip file.", + canResume: false, + }); + } + if (file.size <= 0) { + return setError({ message: "The file is empty.", canResume: false }); + } + const max = status?.maxUploadBytes ?? 0; + if (max && file.size > max) { + return setError({ + message: `The file is too large (max ${formatBytes(max)}).`, + canResume: false, + }); + } + speedRef.current = null; + setProgress({ sent: 0, total: file.size, speed: 0 }); + setPhase("starting"); + try { + uploaderRef.current = await RawUploader.begin(internalId, file, { + onProgress: (sent, total) => { + setProgress({ sent, total, speed: trackSpeed(sent) }); + setPhase((p) => (p === "starting" ? "uploading" : p)); + }, + onFinished: () => { + verifyingIdRef.current = uploaderRef.current?.uploadId ?? null; + setPhase("verifying"); + }, + onError: (message, canResume) => { + setError({ message, canResume }); + if (canResume) { + setPhase("paused"); + } else { + uploaderRef.current = null; + setProgress(null); + setPhase("idle"); + load(); + } + }, + }); + setPhase((p) => (p === "starting" ? "uploading" : p)); + await load(); + onChanged?.(); // readiness now says "upload in progress" + } catch (e: any) { + setProgress(null); + setPhase("idle"); + if (e instanceof RawUploadError && e.code === "ACTIVE_UPLOAD_EXISTS") { + setConflict(e.details?.upload ?? null); + } else { + setError({ + message: e.message || "Could not start the upload.", + canResume: false, + }); + } + } + }; + + const pause = async () => { + await uploaderRef.current?.pause(); + setPhase("paused"); + }; + + const resume = () => { + setError(null); + speedRef.current = null; + setPhase("uploading"); + uploaderRef.current?.resume(); + }; + + // The upload to cancel: the one running here, an interrupted one (after a + // reload), or the one blocking a different file. + const cancelTargetId = + uploaderRef.current?.uploadId || + status?.activeUpload?.uploadId || + conflict?.uploadId || + null; + + const cancel = async () => { + setSuccess(null); + setConfirmCancel(false); + setBusy(true); + try { + if (uploaderRef.current) await uploaderRef.current.cancel(); + else if (cancelTargetId) { + await RawUploadService.cancel(internalId, cancelTargetId); + } + setError(null); + } catch (e: any) { + setError({ + message: e.message || "Could not cancel the upload.", + canResume: false, + }); + } finally { + uploaderRef.current = null; + setConflict(null); + setProgress(null); + setPhase("idle"); + setBusy(false); + await load(); + onChanged?.(); + } + }; + + const toggleExpected = async (expected: boolean) => { + setBusy(true); + try { + await RawUploadService.setExpected(internalId, expected); + await load(); + onChanged?.(); + } catch (e: any) { + setError({ message: e.message, canResume: false }); + } finally { + setBusy(false); + } + }; + + if (!status) { + return ( + + + Raw data (ZIP) + + {loadError ? ( + {loadError} + ) : ( + + )} + + ); + } + + const { current, activeUpload, lastUpload, editable } = status; + const sending = + phase === "starting" || phase === "uploading" || phase === "paused"; + const interrupted = + !!activeUpload && + !uploaderRef.current && + phase === "idle" && + activeUpload.status !== "verifying"; + const canPick = + editable && rawUploadSupported && phase === "idle" && !activeUpload; + const pct = + progress && progress.total + ? Math.floor((progress.sent / progress.total) * 100) + : 0; + const showLastFailure = + phase === "idle" && + !activeUpload && + !error && + lastUpload && + ["failed", "expired"].includes(lastUpload.status); + + return ( + + + Raw data (ZIP) + + + {success && ( + setSuccess(null)} + sx={{ mb: 2 }} + > + Raw data uploaded: {success} + + )} + + toggleExpected(e.target.checked)} + sx={{ "&.Mui-checked": { color: Colors.purple } }} + /> + } + label="This dataset includes raw data" + /> + + {/* Current ZIP */} + {current ? ( + + + Current ZIP: {current.filename || "(unnamed)"} ·{" "} + {formatBytes(current.size)} · uploaded{" "} + {new Date(current.uploadedAt).toLocaleDateString()} + {current.carriedForward + ? " · carried over from the previous version" + : ""} + + + SHA-256{" "} + + {current.sha256.slice(0, 8)}…{current.sha256.slice(-8)} + + ·{" "} + {EXTRACTION_LABEL[current.extractionStatus]} + + + ) : ( + status.rawZipExpected && + !activeUpload && + phase === "idle" && ( + + No raw ZIP uploaded yet. + + ) + )} + + {!rawUploadSupported && ( + + This browser can't upload large files. Please use a recent Chrome, + Firefox, Safari or Edge. + + )} + + {/* Upload interrupted earlier (e.g. the page was reloaded). */} + {interrupted && ( + + An upload of {activeUpload!.filename} ( + {formatBytes(activeUpload!.size)}) was interrupted. Choose the same + file to continue, or cancel it. + {editable && ( + + + + + )} + + )} + + {/* A different file was picked while another upload exists. */} + {conflict && ( + + Another upload ({conflict.filename},{" "} + {formatBytes(conflict.size)}) is in progress for this version. Choose + that same file to continue it, or cancel it to upload a different + file. + + + + + )} + + {/* Progress while sending. */} + {sending && progress && ( + + + {phase === "paused" + ? "Paused" + : phase === "starting" + ? "Starting" + : "Uploading"}{" "} + {uploaderRef.current?.file.name} + + + + {pct}% · {formatBytes(progress.sent)} /{" "} + {formatBytes(progress.total)} + {phase === "uploading" && + progress.speed > 0 && + ` · ${formatBytes(progress.speed)}/s · ${formatEta( + (progress.total - progress.sent) / progress.speed + )}`} + + + {phase === "uploading" && ( + + )} + {phase === "paused" && ( + + )} + + + + )} + + {phase === "verifying" && ( + + + + Verifying the file — checking it and computing its checksum. This + can take a while for very large files. + + + )} + + {error && ( + + {error.message} + + )} + + {showLastFailure && ( + + The last upload ({lastUpload!.filename}) didn't finish:{" "} + {lastUpload!.error || "it expired."} + {current ? " Your current ZIP is unchanged." : ""} + + )} + + {canPick && ( + + )} + {canPick && ( + + ZIP files up to {formatBytes(status.maxUploadBytes)}. + + )} + + {!editable && ( + + Raw data can only be changed while the submission is a draft or has + requested changes. + + )} + + { + const f = e.target.files?.[0]; + e.target.value = ""; // allow picking the same file again later + if (f) startUpload(f); + }} + /> + + setConfirmCancel(false)}> + Cancel this upload? + + + The part already uploaded will be deleted. You can start a new + upload afterwards. + + + + + + + + + ); +}; + +export default RawDataUploader; diff --git a/src/components/User/Dashboard/UploadsTab.tsx b/src/components/User/Dashboard/UploadsTab.tsx new file mode 100644 index 0000000..af8d61d --- /dev/null +++ b/src/components/User/Dashboard/UploadsTab.tsx @@ -0,0 +1,272 @@ +import { CloudUpload, OpenInNew, Visibility } from "@mui/icons-material"; +import { + Alert, + Box, + Button, + Chip, + CircularProgress, + Divider, + List, + ListItem, + ListItemText, + Paper, + Typography, +} from "@mui/material"; +import { Colors } from "design/theme"; +import React, { useEffect, useState } from "react"; +import { useNavigate } from "react-router-dom"; +import { UploadRecord, UploadService } from "services/upload.service"; +import { formatBytes } from "utils/formatBytes"; + +type SubStatus = Exclude; + +const STATUS_CHIP: Record< + SubStatus, + { + label: string; + color?: "default" | "info" | "warning" | "success" | "error"; + sx?: object; + } +> = { + draft: { label: "Draft", color: "default" }, + // Pending uses the theme purple (not MUI's default blue). + pending: { + label: "Pending review", + sx: { backgroundColor: Colors.purple, color: Colors.white }, + }, + changes_requested: { label: "Changes requested", color: "warning" }, + approved: { label: "Approved — awaiting promotion", color: "info" }, + promoted: { label: "Promoted", color: "success" }, + rejected: { label: "Rejected", color: "error" }, +}; + +// Raw ZIP state of the latest version (null = no raw data → no chip). +const rawChip = (r: UploadRecord) => { + if (r.raw_upload_active) { + return { + label: "ZIP uploading", + sx: { color: Colors.purple, borderColor: Colors.purple }, + variant: "outlined" as const, + }; + } + if (r.raw_zip_size != null) { + return { label: `ZIP · ${formatBytes(Number(r.raw_zip_size))}` }; + } + if (r.raw_zip_expected) { + return { + label: "ZIP missing", + color: "warning" as const, + variant: "outlined" as const, + }; + } + return null; +}; + +const UploadsTab: React.FC = () => { + const navigate = useNavigate(); + const [rows, setRows] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + UploadService.listMine() + .then(setRows) + .catch((e) => setError(e.message)) + .finally(() => setLoading(false)); + }, []); + + const formatDate = (d: string | null) => + d + ? new Date(d).toLocaleDateString("en-US", { + year: "numeric", + month: "short", + day: "numeric", + }) + : ""; + + if (loading) { + return ( + + + + ); + } + if (error) { + return ( + + {error} + + ); + } + + return ( + + + + + My Uploads + + + Datasets you have submitted for review. + + + + + + {rows.length === 0 ? ( + + + + No uploads yet + + + Datasets you submit for review will appear here. + + + ) : ( + + + {rows.map((r, index) => { + const chip = r.status ? STATUS_CHIP[r.status] : null; + return ( + + {index > 0 && } + + + + + navigate(`/uploads/${r.internal_id}`) + } + sx={{ + cursor: "pointer", + color: Colors.purple, + "&:hover": { textDecoration: "underline" }, + }} + > + {r.dataset_name || "(unnamed dataset)"} + + + {chip && ( + + )} + {(() => { + const raw = rawChip(r); + return ( + raw && ( + + ) + ); + })()} + + } + secondary={`Submitted ${formatDate(r.created_at)}${ + r.updated_at && r.updated_at !== r.created_at + ? ` · updated ${formatDate(r.updated_at)}` + : "" + }`} + /> + {r.status === "changes_requested" && ( + + Action needed: update and resubmit for review + + )} + + + {r.status === "promoted" && r.promoted_db && r.dataset_id && ( + + )} + {/* All actions (Update, Submit, settings) live on the + upload detail page, which also shows review state. */} + + + + + ); + })} + + + )} + + ); +}; + +export default UploadsTab; diff --git a/src/components/User/UserDashboard.tsx b/src/components/User/UserDashboard.tsx index 42eab48..f8ddeed 100644 --- a/src/components/User/UserDashboard.tsx +++ b/src/components/User/UserDashboard.tsx @@ -3,6 +3,7 @@ import ProfileTab from "./Dashboard/ProfileTab"; import ProjectsTab from "./Dashboard/ProjectsTab"; import SavedDatasetsTab from "./Dashboard/SavedDatasetsTab"; import SecurityTab from "./Dashboard/SecurityTab"; +import UploadsTab from "./Dashboard/UploadsTab"; import LikedDatasetsTab from "./Dashboard/likedDatasetsTab"; import { AccountCircle, @@ -11,6 +12,7 @@ import { Bookmark, Favorite, FolderOpen, + CloudUpload, } from "@mui/icons-material"; import { Box, @@ -56,6 +58,7 @@ const TAB_INDEX: Record = { liked: 3, projects: 4, settings: 5, + uploads: 6, }; const TAB_NAME = Object.fromEntries( @@ -177,6 +180,12 @@ const UserDashboard: React.FC = () => { id="dashboard-tab-2" aria-controls="dashboard-tabpanel-2" /> + } + label="Uploads" + id="dashboard-tab-6" + aria-controls="dashboard-tabpanel-6" + /> @@ -194,6 +203,9 @@ const UserDashboard: React.FC = () => { + + + ); diff --git a/src/design/theme.ts b/src/design/theme.ts index 5a02123..18acfb1 100644 --- a/src/design/theme.ts +++ b/src/design/theme.ts @@ -67,6 +67,28 @@ const theme = createTheme({ fontFamily: "Ubuntu", }, }, + components: { + // Text-field focus uses the theme purple instead of MUI's default blue. + // :not(.Mui-error) keeps the red border/label on invalid fields. + MuiOutlinedInput: { + styleOverrides: { + root: { + "&.Mui-focused:not(.Mui-error) .MuiOutlinedInput-notchedOutline": { + borderColor: Colors.purple, + }, + }, + }, + }, + MuiInputLabel: { + styleOverrides: { + root: { + "&.Mui-focused:not(.Mui-error)": { + color: Colors.purple, + }, + }, + }, + }, + }, }); export default theme; diff --git a/src/pages/UploadDetailPage.tsx b/src/pages/UploadDetailPage.tsx new file mode 100644 index 0000000..da54469 --- /dev/null +++ b/src/pages/UploadDetailPage.tsx @@ -0,0 +1,573 @@ +import ArrowBackIcon from "@mui/icons-material/ArrowBack"; +import DataObjectIcon from "@mui/icons-material/DataObject"; +import EditIcon from "@mui/icons-material/Edit"; +import SendIcon from "@mui/icons-material/Send"; +import VisibilityIcon from "@mui/icons-material/Visibility"; +import { + Alert, + Box, + Button, + Chip, + CircularProgress, + Container, + Divider, + Paper, + TextField, + Typography, +} from "@mui/material"; +import CopyButton from "components/CopyButton"; +import PublishSettingsDialog from "components/Upload/PublishSettingsDialog"; +import RawDataUploader from "components/Upload/RawDataUploader"; +import { Colors } from "design/theme"; +import { useAppSelector } from "hooks/useAppSelector"; +import React, { useCallback, useEffect, useState } from "react"; +import { Link, useNavigate, useParams } from "react-router-dom"; +import { AuthSelector } from "redux/auth/auth.selector"; +import { baseURL } from "services/instance"; +import { + Comment, + UploadDetail, + UploadRecord, + UploadService, +} from "services/upload.service"; +import RoutesEnum from "types/routes.enum"; + +type SubStatus = Exclude; + +const STATUS_CHIP: Record< + SubStatus, + { + label: string; + color?: "default" | "info" | "warning" | "success" | "error"; + sx?: object; + } +> = { + draft: { label: "Draft", color: "default" }, + pending: { + label: "Pending review", + sx: { backgroundColor: Colors.purple, color: Colors.white }, + }, + changes_requested: { label: "Changes requested", color: "warning" }, + approved: { label: "Approved — awaiting promotion", color: "info" }, + promoted: { label: "Promoted", color: "success" }, + rejected: { label: "Rejected", color: "error" }, +}; + +const formatDate = (d: string | null) => + d + ? new Date(d).toLocaleDateString("en-US", { + year: "numeric", + month: "short", + day: "numeric", + }) + : "—"; + +const UploadDetailPage: React.FC = () => { + const { internalId } = useParams<{ internalId: string }>(); + const { isLoggedIn } = useAppSelector(AuthSelector); + const navigate = useNavigate(); + + const [detail, setDetail] = useState(null); + const [comments, setComments] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const [message, setMessage] = useState(""); + const [posting, setPosting] = useState(false); + const [postError, setPostError] = useState(null); + + const [settingsOpen, setSettingsOpen] = useState(false); + + // Review actions (submit / resubmit / withdraw). + const [acting, setActing] = useState(false); + const [actionError, setActionError] = useState(null); + + const runAction = async (fn: (id: string) => Promise) => { + if (!internalId) return; + setActing(true); + setActionError(null); + try { + await fn(internalId); + setDetail(await UploadService.getUpload(internalId)); // refresh status + } catch (e: any) { + setActionError(e.message || "Action failed."); + } finally { + setActing(false); + } + }; + + // Raw ZIP changes affect readiness, so reload the dataset after them. + const reloadDetail = useCallback(async () => { + if (internalId) setDetail(await UploadService.getUpload(internalId)); + }, [internalId]); + + useEffect(() => { + if (!internalId || !isLoggedIn) return; + setLoading(true); + Promise.all([ + UploadService.getUpload(internalId), + UploadService.listComments(internalId), + ]) + .then(([d, c]) => { + setDetail(d); + setComments(c); + }) + .catch((e) => setError(e.message)) + .finally(() => setLoading(false)); + }, [internalId, isLoggedIn]); + + const send = async () => { + const msg = message.trim(); + if (!msg || !internalId) return; + setPosting(true); + setPostError(null); + try { + const created = await UploadService.postComment(internalId, msg); + setComments((prev) => [...prev, created]); + setMessage(""); + } catch (e: any) { + setPostError(e.message || "Failed to post comment."); + } finally { + setPosting(false); + } + }; + + if (!isLoggedIn) { + return ( + + + + Please log in + + + + + ); + } + + if (loading) { + return ( + + + + + + ); + } + + if (error || !detail) { + return ( + + {error || "Dataset not found."} + + + ); + } + + const chip = detail.status ? STATUS_CHIP[detail.status] : null; + // JSON re-uploaded after the reviewer asked for changes? + const updatedSinceReview = + detail.status === "changes_requested" && + !!detail.json_uploaded_at && + !!detail.reviewed_at && + new Date(detail.json_uploaded_at) > new Date(detail.reviewed_at); + const canEditSettings = + (detail.status === "draft" || detail.status === "changes_requested") && + !detail.dataset_id; + + return ( + + + + {/* Header / metadata */} + + + + {detail.dataset_name || "(unnamed dataset)"} + + {chip && ( + + )} + + + + + Public dataset ID:{" "} + {detail.dataset_id ? ( + {detail.dataset_id} + ) : ( + Not yet assigned + )} + + + Internal reference: {detail.internal_id} + + + + Submitted {formatDate(detail.created_at)} + {detail.updated_at && detail.updated_at !== detail.created_at + ? ` · updated ${formatDate(detail.updated_at)}` + : ""} + + + + + {/* Locked while pending/approved; rejected/promoted use Update to + start a new version. */} + {detail.status !== "pending" && detail.status !== "approved" && ( + + )} + {/* Opens the stored doc via our owner-checked backend proxy (the + sandbox CouchDB URL is never exposed to the browser). */} + + {detail.status === "promoted" && + detail.promoted_db && + detail.dataset_id && ( + + )} + + + + {/* Publishing settings: editable while draft/changes_requested and not + yet published (the public URL is fixed once a public ID exists). */} + + + Publishing settings + {canEditSettings && ( + + )} + + + + Database: {detail.requested_db || "public"} + {(detail.requested_db || "public") === "public" ? " (default)" : ""} + + + Dataset ID:{" "} + {detail.requested_dataset_id ? ( + <> + {detail.requested_dataset_id} (preferred) + + ) : ( + Assigned by NeuroJSON after approval + )} + + {detail.dataset_id && ( + + Locked — this dataset is published, so its address can't change. + + )} + + + + setSettingsOpen(false)} + onSaved={async () => { + setSettingsOpen(false); + if (internalId) setDetail(await UploadService.getUpload(internalId)); + }} + /> + + {/* Raw data (ZIP): optional; uploads go straight to the storage API */} + + + {/* Review: readiness checklist + submit / resubmit / withdraw */} + + + Review + + + + {detail.json_status === "failed" + ? `✗ ${detail.json_error || "The last JSON update failed."}` + : `✓ JSON uploaded${ + detail.json_uploaded_at + ? ` (${new Date(detail.json_uploaded_at).toLocaleString()})` + : "" + }`} + + + {(detail.status === "draft" || + detail.status === "changes_requested") && ( + <> + {/* Changes requested: remind the user that updating is not enough — + they must click Resubmit. Wording depends on whether the JSON + was updated after the reviewer's decision. */} + {detail.status === "changes_requested" && ( + + {updatedSinceReview ? ( + <> + You've updated your dataset since the reviewer's request. + Click Resubmit for review so the reviewer + can see your changes. + + ) : ( + <> + The reviewer requested changes. Read their message below, + update your files, then click{" "} + Resubmit for review. + + )} + + )} + {!detail.readiness.canSubmit && ( + + {detail.readiness.problems.join(" ")} + + )} + + + )} + + {detail.status === "pending" && ( + + + Waiting for review + {detail.submitted_at + ? ` · submitted ${new Date( + detail.submitted_at + ).toLocaleString()}` + : ""} + . Withdraw it to make changes. + + + + )} + + {detail.status === "approved" && ( + + Approved and waiting to be published. It can't be changed now. + + )} + + {actionError && ( + + {actionError} + + )} + + + {/* Conversation */} + + + Review discussion + + + {comments.length === 0 ? ( + + No messages yet. Use the box below to reply to the review team. + + ) : ( + + {comments.map((c) => ( + + + + {c.is_owner ? "You" : "Reviewer"} ·{" "} + {new Date(c.created_at).toLocaleString()} + + + {c.message} + + + + ))} + + )} + + + + setMessage(e.target.value)} + multiline + minRows={2} + fullWidth + /> + {postError && ( + + {postError} + + )} + + + + + + ); +}; + +export default UploadDetailPage; diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx new file mode 100644 index 0000000..e3f5213 --- /dev/null +++ b/src/pages/UploadPage.tsx @@ -0,0 +1,512 @@ +import CloudUploadIcon from "@mui/icons-material/CloudUpload"; +import { + Alert, + Box, + Button, + CircularProgress, + Container, + Dialog, + DialogActions, + DialogContent, + DialogContentText, + DialogTitle, + Link as MuiLink, + Paper, + TextField, + Typography, +} from "@mui/material"; +import CopyButton from "components/CopyButton"; +import { Colors } from "design/theme"; +import { useAppSelector } from "hooks/useAppSelector"; +import React, { useRef, useState } from "react"; +import { Link, useLocation, useNavigate } from "react-router-dom"; +import { AuthSelector } from "redux/auth/auth.selector"; +import { UploadService } from "services/upload.service"; +import RoutesEnum from "types/routes.enum"; +import { + getRequestedDbError, + getRequestedIdError, + useExistingDbNames, +} from "utils/uploadValidation"; + +const UploadPage: React.FC = () => { + const { isLoggedIn } = useAppSelector(AuthSelector); + const fileInputRef = useRef(null); + const location = useLocation(); + const navigate = useNavigate(); + const initialInternalId = + new URLSearchParams(location.search).get("internalId") ?? ""; + + const [file, setFile] = useState(null); + const [parsed, setParsed] = useState | null>(null); + const [parseError, setParseError] = useState(null); + const [internalId, setInternalId] = useState(initialInternalId); + const [datasetName, setDatasetName] = useState(""); + const [requestedDatasetId, setRequestedDatasetId] = useState(""); + const [requestedDb, setRequestedDb] = useState(""); + // Existing public db names — typing one of these is blocked (shared helper). + const existingDbs = useExistingDbNames(); + const [submitting, setSubmitting] = useState(false); + const [result, setResult] = useState<{ + internalId: string; + submissionId: string; + status: string; + } | null>(null); + const [submitError, setSubmitError] = useState(null); + const [infoMessage, setInfoMessage] = useState(null); // approved/blocked + const [confirmPrompt, setConfirmPrompt] = useState<{ + code: string; + message: string; + } | null>(null); + + const resetOutcome = () => { + setResult(null); + setSubmitError(null); + setInfoMessage(null); + }; + + const handleFile = async (f: File | undefined) => { + resetOutcome(); + setParsed(null); + setParseError(null); + if (!f) return; + setFile(f); + try { + const text = await f.text(); + const json = JSON.parse(text); + if (!json || typeof json !== "object" || Array.isArray(json)) { + setParseError("The file must contain a single JSON object."); + return; + } + setParsed(json); + } catch (e: any) { + setParseError(`Not valid JSON: ${e.message}`); + } + }; + + const submit = async (confirm: boolean) => { + if (!parsed) return; + setSubmitting(true); + resetOutcome(); + setConfirmPrompt(null); + try { + const { status, data } = await UploadService.uploadJson(parsed, { + internalId: internalId.trim() || undefined, + datasetName: datasetName.trim() || undefined, + // Only sent for a new dataset (the backend ignores them on update). + requestedDatasetId: internalId.trim() + ? undefined + : requestedDatasetId.trim() || undefined, + requestedDb: internalId.trim() + ? undefined + : requestedDb.trim() || undefined, + confirm, + }); + if (status === 201) { + setResult({ + internalId: data.internal_id, + submissionId: data.submission_id, + status: data.status, + }); + } else if (status === 409 && data.requiresConfirmation) { + setConfirmPrompt({ code: data.code, message: data.message }); + } else if (status === 409 && data.message) { + // Informational block (e.g. approved, awaiting promotion). + setInfoMessage(data.message); + } else { + // Real errors, incl. REQUESTED_ID_TAKEN (which carries `error`). + setSubmitError( + data.error || data.message || `Upload failed (${status})` + ); + } + } catch (e: any) { + setSubmitError(e.message || "Upload failed."); + } finally { + setSubmitting(false); + } + }; + + // Login guard — the endpoint requires auth; prompt to log in. + if (!isLoggedIn) { + return ( + + + + Please log in to upload + + + Uploading a dataset for review requires a NeuroJSON account. + + + + + ); + } + + // If the file already carries a name, use it and disable the name field. + // Mirrors the backend: Name must be text; spaces-only counts as no name. + const dd = parsed?.["dataset_description.json"] as any; + const rawName = + dd && typeof dd === "object" && !Array.isArray(dd) ? dd.Name : undefined; + const fileNameTypeError = + rawName !== undefined && rawName !== null && typeof rawName !== "string" + ? 'The "Name" in dataset_description.json must be text.' + : ""; + const nameFromFile = typeof rawName === "string" ? rawName.trim() : ""; + const typedName = datasetName.trim(); + const datasetNameError = fileNameTypeError + ? fileNameTypeError + : (nameFromFile || typedName).length > 255 + ? "Too long — 255 characters max." + : ""; + const hasName = !!nameFromFile || !!typedName; + + // Publishing settings are set here only for a NEW dataset; for an update + // they're edited on the upload detail page (PATCH /settings). + const isNewDataset = !internalId.trim(); + + // Live validation (shared with the detail-page settings dialog). + const requestedIdError = isNewDataset + ? getRequestedIdError(requestedDatasetId) + : ""; + const requestedDbError = isNewDataset + ? getRequestedDbError(requestedDb, existingDbs) + : ""; + + return ( + + + Upload a dataset + + + Select a JSON file. It is saved as a draft in our sandbox. When + everything is ready, submit it for review from its upload detail page + (Dashboard → Uploads). + + + + + Uploading does not send your dataset for review. Review starts only + after you click Submit for review on the upload + detail page; the NeuroJSON team reviews it before it is added to a + database. A new reference id is assigned on upload — any existing{" "} + _id/_rev in your file is ignored. Raw data + (a ZIP) is added afterwards, on the upload detail page. + + + {/* Updating an existing dataset: the id comes only from the URL + (?internalId=…, set by the Update button) — never typed by hand. */} + {!isNewDataset && ( + + You're updating an existing dataset (reference{" "} + {internalId}).{" "} + { + setInternalId(""); + resetOutcome(); + navigate("/upload", { replace: true }); + }} + sx={{ + color: Colors.purple, + fontWeight: 600, + verticalAlign: "baseline", + }} + > + Start a new dataset instead + + + )} + + handleFile(e.target.files?.[0])} + /> + + {file && ( + + Selected: {file.name}{" "} + {parsed && ( + + ✓ valid JSON + + )} + + )} + + {parseError && ( + + {parseError} + + )} + + {/* Dataset name — used when the file has no dataset_description.json Name */} + setDatasetName(e.target.value)} + disabled={!!nameFromFile || !!fileNameTypeError} + size="small" + fullWidth + error={!!datasetNameError} + sx={{ + mt: 2.5, + // Highlight the "no name found" prompt so it's easy to notice + // (red error styling wins when there's a problem). + "& .MuiFormHelperText-root": + nameFromFile || datasetNameError + ? {} + : { color: Colors.purple, fontWeight: 600 }, + }} + helperText={ + datasetNameError || + (nameFromFile + ? 'Name taken from your file ("dataset_description.json" → Name).' + : 'No name found in your file ("dataset_description.json" → Name) — enter one here.') + } + /> + + {/* Publishing settings: only for a NEW dataset. For an update they're + edited on the upload detail page. */} + {!isNewDataset && ( + + Publishing settings (database and dataset ID) are changed on the + upload detail page. + + )} + + {isNewDataset && ( + <> + {/* Target database — where the dataset is published after review */} + setRequestedDb(e.target.value)} + size="small" + fullWidth + error={!!requestedDbError} + helperText={requestedDbError || undefined} + sx={{ mt: 2.5 }} + /> + +
  • + Becomes part of your dataset’s public URL after review (e.g.{" "} + /db/smith-lab/…). +
    • +
  • + Leave blank for the default public database. +
    • +     + + {/* Optional preferred public ID (a preference; final id is set at review) */} + setRequestedDatasetId(e.target.value)} + size="small" + fullWidth + error={!!requestedIdError} + helperText={requestedIdError || undefined} + sx={{ mt: 2.5 }} + /> + +
  • + Becomes part of your dataset’s public URL after review (e.g.{" "} + /db/…/my-fmri-study). +
    • +
  • + Use only lowercase letters and numbers; join words with - or _ + (e.g. my-fmri-study). +
    • +     + + )} + + + + + + {result && ( + + {result.status === "changes_requested" + ? "Saved. Resubmit it for review when it's ready." + : "Saved as draft. Submit it for review when it's ready."} +
    + Internal reference: {result.internalId} + +
    + The public dataset ID is assigned after your dataset passes review. +
    + + Open upload details → + +     + )} + {infoMessage && ( + + {infoMessage} + + )} + {submitError && ( + + {submitError} + + )} +     + + + Coming from AutoBIDSify? Convert your dataset in the{" "} + + converter + + , then upload the generated JSON here. + + + {/* Confirmation dialog for promoted/rejected resubmissions */} + setConfirmPrompt(null)}> + Submit a new version? + + {confirmPrompt?.message} + {confirmPrompt && ( + + {confirmPrompt.code === "DATASET_ALREADY_PROMOTED" + ? "This starts a new draft version and replaces your working copy in the sandbox. The version already published to the public database stays unchanged — it is only replaced if the new version passes review." + : "This starts a new draft version and replaces your working copy in the sandbox. Submit it for review when it's ready."} + + )} + + + + + + +     + ); +}; + +export default UploadPage; diff --git a/src/services/rawUpload.service.ts b/src/services/rawUpload.service.ts new file mode 100644 index 0000000..ef3dd50 --- /dev/null +++ b/src/services/rawUpload.service.ts @@ -0,0 +1,136 @@ +import { baseURL } from "services/instance"; + +// Calls to REN's raw ZIP routes (/api/v1/uploads/:internalId/raw…). The file +// itself never goes through REN — the browser sends it to the storage API +// with tus (see utils/rawUploader.ts), using the token these routes return. + +export type RawAttemptStatus = + | "initiated" + | "uploading" + | "verifying" + | "complete" + | "failed" + | "cancelled" + | "expired"; + +export type RawExtractionStatus = + | "not_extracted" + | "extracting" + | "extracted" + | "failed"; + +// One upload attempt (the active one, or the last failed/cancelled/expired one). +export interface RawUploadSummary { + uploadId: string; + status: RawAttemptStatus; + filename: string | null; + size: number; + error: string | null; + createdAt: string; + updatedAt: string; + completedAt: string | null; +} + +// The ZIP this version currently uses. +export interface RawCurrentZip { + rawId: string; + filename: string | null; + size: number; + sha256: string; + extractionStatus: RawExtractionStatus; + carriedForward: boolean; + uploadedAt: string; +} + +export interface RawStatus { + rawZipExpected: boolean; + editable: boolean; + maxUploadBytes: number; // server's per-file limit (bytes) + current: RawCurrentZip | null; + activeUpload: RawUploadSummary | null; + lastUpload: RawUploadSummary | null; +} + +// What REN returns when an upload starts (or resumes). +export interface RawUploadSession { + uploadId: string; + endpoint: string; // storage API tus endpoint, e.g. https://…/neurojson-storage/files + token: string; // short-lived upload token (memory only — never stored) + expiresAt: string; + chunkSize: number; + resumed: boolean; // true = continue the existing upload for this file +} + +export interface RawUploadToken { + token: string; + expiresAt: string; +} + +// Error from a raw route that keeps REN's code (e.g. ACTIVE_UPLOAD_EXISTS, +// FILE_TOO_LARGE, NOT_EDITABLE) and extra fields (e.g. the other upload), so +// the UI can react to each case instead of only showing a message. +export class RawUploadError extends Error { + status: number; + code?: string; + details: any; + + constructor(message: string, status: number, code?: string, details?: any) { + super(message); + this.name = "RawUploadError"; + this.status = status; + this.code = code; + this.details = details; + } +} + +const request = async ( + method: string, + path: string, + body?: unknown +): Promise => { + const res = await fetch(`${baseURL}/uploads${path}`, { + method, + credentials: "include", + headers: + body !== undefined ? { "Content-Type": "application/json" } : undefined, + body: body !== undefined ? JSON.stringify(body) : undefined, + }); + const data = await res.json().catch(() => ({})); + if (!res.ok) { + throw new RawUploadError( + data.error || `Request failed (${res.status})`, + res.status, + data.code, + data + ); + } + return data as T; +}; + +export const RawUploadService = { + // Current ZIP + active/last upload for the dataset's latest version. + getStatus: (internalId: string) => + request("GET", `/${internalId}/raw`), + + // Start an upload, or get the active one back (same name + size → resumed). + start: (internalId: string, file: { name: string; size: number }) => + request("POST", `/${internalId}/raw`, { + filename: file.name, + size: file.size, + }), + + // Fresh token for a long upload (tokens last 15 minutes). + refreshToken: (internalId: string, uploadId: string) => + request("POST", `/${internalId}/raw/${uploadId}/token`), + + cancel: (internalId: string, uploadId: string) => + request<{ status: string }>("DELETE", `/${internalId}/raw/${uploadId}`), + + // "This dataset includes raw data" checkbox. + setExpected: (internalId: string, expected: boolean) => + request<{ rawZipExpected: boolean }>( + "PATCH", + `/${internalId}/raw/expected`, + { expected } + ), +}; diff --git a/src/services/upload.service.ts b/src/services/upload.service.ts new file mode 100644 index 0000000..a59ff61 --- /dev/null +++ b/src/services/upload.service.ts @@ -0,0 +1,177 @@ +import { baseURL } from "services/instance"; + +export interface UploadResponse { + status: number; // HTTP status + data: any; // parsed JSON body +} + +export interface UploadRecord { + internal_id: string; // permanent logical-dataset id (also the sandbox _id) + dataset_id: string | null; // final public id — null until promoted + requested_dataset_id: string | null; + dataset_name: string | null; + submission_id: string | null; + status: + | "draft" + | "pending" + | "changes_requested" + | "approved" + | "rejected" + | "promoted" + | null; + created_at: string | null; + updated_at: string | null; + promoted_db: string | null; + promoted_at: string | null; + raw_zip_expected: boolean | null; + raw_zip_size: string | number | null; // BIGINT arrives as a string + raw_upload_active: boolean | null; +} + +export interface UploadDetail { + internal_id: string; + dataset_id: string | null; + requested_dataset_id: string | null; + requested_db: string | null; + dataset_name: string | null; + submission_id: string | null; + status: UploadRecord["status"]; + created_at: string | null; + updated_at: string | null; + promoted_db: string | null; + promoted_at: string | null; + json_status: "uploaded" | "failed" | null; + json_uploaded_at: string | null; + json_error: string | null; + submitted_at: string | null; + reviewed_at: string | null; + raw_zip_expected: boolean; + readiness: { canSubmit: boolean; problems: string[] }; +} + +export interface Comment { + id: number; + username: string; // owner's real name, or "Reviewer" (identity masked) + message: string; + created_at: string; + is_owner: boolean; +} + +// POST /api/v1/uploads — submit a JSON dataset for review. fetch with +// credentials:"include" sends the httpOnly auth cookie (the shared axios +// instance has withCredentials:false). Returns status + body so the caller +// branches on 201 (ok) / 409 (confirmation or blocked) / 400 / 403 itself. +export const UploadService = { + uploadJson: async ( + doc: unknown, + opts?: { + internalId?: string; + datasetName?: string; + requestedDatasetId?: string; + requestedDb?: string; + confirm?: boolean; + } + ): Promise => { + const params = new URLSearchParams(); + if (opts?.internalId) params.set("internalId", opts.internalId); + if (opts?.datasetName) params.set("datasetName", opts.datasetName); + if (opts?.requestedDatasetId) + params.set("requestedDatasetId", opts.requestedDatasetId); + if (opts?.requestedDb) params.set("requestedDb", opts.requestedDb); + if (opts?.confirm) params.set("confirm", "true"); + const qs = params.toString() ? `?${params.toString()}` : ""; + + const response = await fetch(`${baseURL}/uploads${qs}`, { + method: "POST", + credentials: "include", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(doc), + }); + const data = await response.json().catch(() => ({})); + return { status: response.status, data }; + }, + + listMine: async (): Promise => { + const res = await fetch(`${baseURL}/uploads`, { credentials: "include" }); + if (!res.ok) throw new Error(`Failed to load uploads (${res.status})`); + return res.json(); + }, + + getUpload: async (internalId: string): Promise => { + const res = await fetch(`${baseURL}/uploads/${internalId}`, { + credentials: "include", + }); + if (!res.ok) { + const body = await res.json().catch(() => ({})); + throw new Error(body.error || `Failed to load dataset (${res.status})`); + } + return res.json(); + }, + + listComments: async (internalId: string): Promise => { + const res = await fetch(`${baseURL}/uploads/${internalId}/comments`, { + credentials: "include", + }); + if (!res.ok) throw new Error(`Failed to load comments (${res.status})`); + return res.json(); + }, + + postComment: async (internalId: string, message: string): Promise => { + const res = await fetch(`${baseURL}/uploads/${internalId}/comments`, { + method: "POST", + credentials: "include", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ message }), + }); + if (!res.ok) { + const body = await res.json().catch(() => ({})); + throw new Error(body.error || `Failed to post comment (${res.status})`); + } + return res.json(); + }, + + // draft / changes_requested → pending + submitForReview: async (internalId: string): Promise<{ status: string }> => { + const res = await fetch(`${baseURL}/uploads/${internalId}/submit`, { + method: "POST", + credentials: "include", + }); + if (!res.ok) { + const body = await res.json().catch(() => ({})); + throw new Error(body.error || `Failed to submit (${res.status})`); + } + return res.json(); + }, + + // Publishing settings. A key left out = unchanged; null = reset + // (requestedDatasetId → NeuroJSON assigns; requestedDb → "public"). + updateSettings: async ( + internalId: string, + settings: { requestedDb?: string | null; requestedDatasetId?: string | null } + ): Promise<{ requested_dataset_id: string | null; requested_db: string }> => { + const res = await fetch(`${baseURL}/uploads/${internalId}/settings`, { + method: "PATCH", + credentials: "include", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(settings), + }); + if (!res.ok) { + const body = await res.json().catch(() => ({})); + throw new Error(body.error || `Failed to update settings (${res.status})`); + } + return res.json(); + }, + + // pending → draft (only before the reviewer acts) + withdraw: async (internalId: string): Promise<{ status: string }> => { + const res = await fetch(`${baseURL}/uploads/${internalId}/withdraw`, { + method: "POST", + credentials: "include", + }); + if (!res.ok) { + const body = await res.json().catch(() => ({})); + throw new Error(body.error || `Failed to withdraw (${res.status})`); + } + return res.json(); + }, +}; diff --git a/src/types/routes.enum.ts b/src/types/routes.enum.ts index e8c2658..d7aa2a3 100644 --- a/src/types/routes.enum.ts +++ b/src/types/routes.enum.ts @@ -5,5 +5,6 @@ enum RoutesEnum { ABOUT = "/about", // New route for the about page DASHBOARD = "/dashboard", BIDS_CONVERTER = "/autobidsify", + UPLOAD = "/upload", // logged-in users submit a JSON dataset for review } export default RoutesEnum; diff --git a/src/utils/formatBytes.ts b/src/utils/formatBytes.ts new file mode 100644 index 0000000..6be73d1 --- /dev/null +++ b/src/utils/formatBytes.ts @@ -0,0 +1,14 @@ +// Human-readable size in DECIMAL units (1 GB = 1,000,000,000 bytes), matching +// the "50 GB max" limit message. +export const formatBytes = (bytes: number): string => { + if (!Number.isFinite(bytes) || bytes < 0) return "—"; + if (bytes < 1000) return `${bytes} B`; + const units = ["KB", "MB", "GB", "TB"]; + let value = bytes / 1000; + let i = 0; + while (value >= 1000 && i < units.length - 1) { + value /= 1000; + i++; + } + return `${value.toFixed(value < 10 ? 1 : 0)} ${units[i]}`; +}; diff --git a/src/utils/rawUploader.ts b/src/utils/rawUploader.ts new file mode 100644 index 0000000..677d3cf --- /dev/null +++ b/src/utils/rawUploader.ts @@ -0,0 +1,173 @@ +import { + RawUploadError, + RawUploadService, + RawUploadSession, +} from "services/rawUpload.service"; +import * as tus from "tus-js-client"; + +// Sends one raw ZIP from the browser straight to the storage API with tus, +// resumably, in chunks. REN only hands out the session and short-lived +// tokens; the token is kept in memory (never in localStorage). + +export interface RawUploaderCallbacks { + onProgress?: (sent: number, total: number) => void; + onFinished?: () => void; // every byte stored — the server now verifies it + onError?: (message: string, canResume: boolean) => void; +} + +const REFRESH_MARGIN_MS = 3 * 60 * 1000; // renew tokens with < 3 min left +// Statuses where retrying the same request can't help. +const STOP_STATUSES = new Set([400, 403, 404, 409, 410, 413, 507]); + +export const rawUploadSupported = tus.isSupported; + +// Turn any upload error into a short message + whether Resume makes sense. +const describeError = (err: any): { message: string; canResume: boolean } => { + const cause = err?.causingError ?? err; + if (cause instanceof RawUploadError) { + // A token refresh with REN failed. + if (cause.status === 401) { + return { + message: + "Your login session ended. Log in again, then choose the same file to continue.", + canResume: false, + }; + } + return { message: cause.message, canResume: false }; + } + const status: number = err?.originalResponse?.getStatus?.() ?? 0; + const body: string = (err?.originalResponse?.getBody?.() || "").trim(); + if (STOP_STATUSES.has(status)) { + return { + message: body || `The upload was refused (HTTP ${status}).`, + canResume: false, + }; + } + if (status === 401) { + return { + message: "The upload permission expired. Click Resume to continue.", + canResume: true, + }; + } + return { + message: "The connection was interrupted. Click Resume to continue.", + canResume: true, + }; +}; + +export class RawUploader { + readonly internalId: string; + readonly file: File; + private session: RawUploadSession; + private callbacks: RawUploaderCallbacks; + private upload: tus.Upload | null = null; + private token: string; + private tokenExpiresAt: number; + + private constructor( + internalId: string, + file: File, + session: RawUploadSession, + callbacks: RawUploaderCallbacks + ) { + this.internalId = internalId; + this.file = file; + this.session = session; + this.callbacks = callbacks; + this.token = session.token; + this.tokenExpiresAt = Date.parse(session.expiresAt); + } + + // Ask REN for a session (new, or the existing one for this same file) and + // start sending. Throws RawUploadError, e.g. ACTIVE_UPLOAD_EXISTS when a + // DIFFERENT file is already being uploaded for this version. + static async begin( + internalId: string, + file: File, + callbacks: RawUploaderCallbacks + ): Promise { + const session = await RawUploadService.start(internalId, { + name: file.name, + size: file.size, + }); + const uploader = new RawUploader(internalId, file, session, callbacks); + uploader.start(); + return uploader; + } + + get uploadId() { + return this.session.uploadId; + } + + get resumed() { + return this.session.resumed; + } + + private async currentToken(): Promise { + if (Date.now() < this.tokenExpiresAt - REFRESH_MARGIN_MS) return this.token; + const fresh = await RawUploadService.refreshToken( + this.internalId, + this.session.uploadId + ); + this.token = fresh.token; + this.tokenExpiresAt = Date.parse(fresh.expiresAt); + return this.token; + } + + private start() { + const { endpoint, uploadId, chunkSize, resumed } = this.session; + this.upload = new tus.Upload(this.file, { + endpoint, + // The upload id is known, so resume exactly that upload: tus asks the + // server for its offset and continues. (If the server has nothing yet, + // tus creates it — same id, since the server names uploads by token.) + uploadUrl: resumed ? `${endpoint}/${uploadId}` : null, + chunkSize, + metadata: { + filename: this.file.name, // display only; never used for paths + filetype: this.file.type || "application/zip", + }, + storeFingerprintForResuming: false, // we resume by upload id instead + retryDelays: [0, 1000, 3000, 5000, 10000, 20000, 30000, 60000], + onBeforeRequest: async (req) => { + req.setHeader("Authorization", `Bearer ${await this.currentToken()}`); + }, + onShouldRetry: (err) => { + const status = err.originalResponse?.getStatus() ?? 0; + if (status === 401) { + this.tokenExpiresAt = 0; // next attempt fetches a fresh token + return true; + } + return !STOP_STATUSES.has(status); // network errors, 5xx, 423 → retry + }, + onProgress: (sent, total) => this.callbacks.onProgress?.(sent, total), + onSuccess: () => this.callbacks.onFinished?.(), + onError: (err) => { + const { message, canResume } = describeError(err); + this.callbacks.onError?.(message, canResume); + }, + }); + this.upload.start(); + } + + // Stop sending; the server keeps what it has. + async pause() { + await this.upload?.abort(false); + } + + // Continue from the server's offset (tus re-checks it first). + resume() { + this.upload?.start(); + } + + // Stop, ask the storage API to drop the partial data, and tell REN. If the + // storage call fails, REN's cleanup job removes the leftovers anyway. + async cancel() { + try { + await this.upload?.abort(true); + } catch { + // already finished or not created yet — REN's cancel still applies + } + await RawUploadService.cancel(this.internalId, this.session.uploadId); + } +} diff --git a/src/utils/uploadValidation.ts b/src/utils/uploadValidation.ts new file mode 100644 index 0000000..5550fab --- /dev/null +++ b/src/utils/uploadValidation.ts @@ -0,0 +1,71 @@ +import { useEffect, useState } from "react"; +import { NeurojsonService } from "services/neurojson.service"; + +// Live checks for the upload publishing settings. They mirror the backend +// rules (^[a-z0-9][a-z0-9_-]{2,62}$; no "njds" prefix for dataset ids) but +// give one specific message per problem. The backend stays authoritative. + +const CHAR_RE = /^[a-z0-9][a-z0-9_-]*$/; // dataset id: allowed chars + first-char rule +const DB_CHAR_RE = /^[a-z][a-z0-9_-]*$/; // CouchDB: db names start with a letter + +const lengthError = (v: string) => + v.length < 3 + ? "Too short — use at least 3 characters." + : v.length > 63 + ? "Too long — use 63 characters or fewer." + : ""; + +// Preferred public dataset id. Empty = no preference (valid). +export const getRequestedIdError = (raw: string): string => { + const v = raw.trim(); + if (!v) return ""; + if (v.startsWith("njds")) { + return 'Cannot start with "njds" (reserved for NeuroJSON-assigned IDs).'; + } + if (!CHAR_RE.test(v)) { + return "Use only lowercase letters and numbers; join words with - or _ (e.g. my-fmri-study). Cannot start with - or _."; + } + return lengthError(v); +}; + +// Target database. Empty = default "public" (valid). Names of existing +// databases belong to other collections, so they're blocked (except public). +export const getRequestedDbError = ( + raw: string, + existingDbs: Set +): string => { + const v = raw.trim(); + if (!v) return ""; + if (!/^[a-z]/.test(v)) { + return "Must start with a lowercase letter (e.g. smith-lab)."; + } + if (!DB_CHAR_RE.test(v)) { + return "Use only lowercase letters and numbers; join words with - or _ (e.g. smith-lab)."; + } + const len = lengthError(v); + if (len) return len; + if (v !== "public" && existingDbs.has(v)) { + return `A database named "${v}" already exists. Choose a different name or leave blank for public.`; + } + return ""; +}; + +// Existing public db names (lowercased), fetched for the collision check only — +// never offered as a picker. Non-fatal if the registry can't be loaded. +export const useExistingDbNames = (): Set => { + const [names, setNames] = useState>(new Set()); + useEffect(() => { + NeurojsonService.getRegistry() + .then((data: any) => + setNames( + new Set( + (data?.database || []).map((d: any) => + String(d.name || "").toLowerCase() + ) + ) + ) + ) + .catch(() => {}); + }, []); + return names; +}; diff --git a/storage-api/.env.example b/storage-api/.env.example new file mode 100644 index 0000000..95c5e6b --- /dev/null +++ b/storage-api/.env.example @@ -0,0 +1,22 @@ +# Where Node listens (Apache proxies to it in prod) +HOST=127.0.0.1 +PORT=8090 +BASE_PATH=/neurojson-storage + +# Storage root. Dev on the Mac: ./storage-dev ; Zodiac: /local_mount/space/zodiac/1/users/neurojson/uploads +UPLOAD_ROOT=./storage-dev + +# Browser origins allowed to upload (comma separated) +ALLOWED_ORIGINS=http://localhost:3000 + +# REN main API for pushes. EMPTY = pushes off (dev: REN pulls instead). +MAIN_API_URL= + +# From the "Zodiac" block of backend/scripts/generate-upload-keys.js (same run as REN's keys) +UPLOAD_TOKEN_PUBLIC_KEYS='{"dev-2026-09":"-----BEGIN PUBLIC KEY-----\n...\n-----END PUBLIC KEY-----"}' +STORAGE_HMAC_ACTIVE_KID=dev-2026-09 +STORAGE_HMAC_KEYS='{"dev-2026-09":"..."}' + +# Limits +MAX_UPLOAD_BYTES=50000000000 +DISK_RESERVE_PERCENT=5 diff --git a/storage-api/package-lock.json b/storage-api/package-lock.json new file mode 100644 index 0000000..02d6d82 --- /dev/null +++ b/storage-api/package-lock.json @@ -0,0 +1,1281 @@ +{ + "name": "neurojson-storage-api", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "neurojson-storage-api", + "version": "0.1.0", + "dependencies": { + "@tus/file-store": "^2.1.1", + "@tus/server": "^2.4.5", + "dotenv": "^17.2.3", + "express": "^5.1.0" + }, + "devDependencies": { + "tus-js-client": "^4.3.1" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "license": "MIT", + "optional": true + }, + "node_modules/@redis/client": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-5.12.1.tgz", + "integrity": "sha512-7aPGWeqA3uFm43o19umzdl16CEjK/JQGtSXVPevplTaOU3VJA/rseBC1QvYUz9lLDIMBimc4SW/zrW4S89BaCA==", + "license": "MIT", + "optional": true, + "dependencies": { + "cluster-key-slot": "1.1.2" + }, + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@node-rs/xxhash": "^1.1.0", + "@opentelemetry/api": ">=1 <2" + }, + "peerDependenciesMeta": { + "@node-rs/xxhash": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + } + } + }, + "node_modules/@tus/file-store": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@tus/file-store/-/file-store-2.1.1.tgz", + "integrity": "sha512-/uXV3Ibb4Upirrsj7lSrX+o08H7IKYEkPHV8TKAPPK6CYUcOMTilluOkwPrb9JZydLSYlIBQkhZZVS9sPEfDwQ==", + "license": "MIT", + "dependencies": { + "@tus/utils": "^0.7.1", + "debug": "^4.3.4" + }, + "engines": { + "node": ">=20.19.0" + }, + "optionalDependencies": { + "@redis/client": "^5.0.0" + } + }, + "node_modules/@tus/server": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/@tus/server/-/server-2.4.5.tgz", + "integrity": "sha512-DaQY5F1/JC1E+eDNY+Q5L08wgQ8NHFDNW/E6NclSx10BOHJk85BT4mSId49Ep890Utsk+6hEEuhNFb/LH+Og0w==", + "license": "MIT", + "dependencies": { + "@tus/utils": "^0.7.1", + "debug": "^4.3.4", + "lodash.throttle": "^4.1.1", + "set-cookie-parser": "^2.7.1", + "srvx": "~0.11.15" + }, + "engines": { + "node": ">=20.19.0" + }, + "optionalDependencies": { + "@redis/client": "^5.0.0", + "ioredis": "^5.4.1" + } + }, + "node_modules/@tus/utils": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@tus/utils/-/utils-0.7.1.tgz", + "integrity": "sha512-KXKK6mhsRto7cApRtJFoIovd6rAXJBUd1qR6S1ynVXApdd+jT1eBHprKP0D9oZtJwwD/B8+n3DLAm68DYPqfuQ==", + "license": "MIT", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/combine-errors": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/combine-errors/-/combine-errors-3.0.3.tgz", + "integrity": "sha512-C8ikRNRMygCwaTx+Ek3Yr+OuZzgZjduCOfSQBjbM8V3MfgcjSTeto/GXP6PAwKvJz/v15b7GHZvx5rOlczFw/Q==", + "dev": true, + "dependencies": { + "custom-error-instance": "2.1.1", + "lodash.uniqby": "4.5.0" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/custom-error-instance": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/custom-error-instance/-/custom-error-instance-2.1.1.tgz", + "integrity": "sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg==", + "dev": true, + "license": "ISC" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dotenv": { + "version": "17.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", + "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "license": "MIT", + "optional": true, + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/js-base64": { + "version": "3.9.4", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.9.4.tgz", + "integrity": "sha512-PtOMXpEGuP0RRiRXsjzHzl44dMHxSu2CPvAhinupR1tBa88me+1DPqsobl7eupn5UukjLkln1ZnAOAOXOrYG0Q==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/lodash._baseiteratee": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash._baseiteratee/-/lodash._baseiteratee-4.7.0.tgz", + "integrity": "sha512-nqB9M+wITz0BX/Q2xg6fQ8mLkyfF7MU7eE+MNBNjTHFKeKaZAPEzEg+E8LWxKWf1DQVflNEn9N49yAuqKh2mWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash._stringtopath": "~4.8.0" + } + }, + "node_modules/lodash._basetostring": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-4.12.0.tgz", + "integrity": "sha512-SwcRIbyxnN6CFEEK4K1y+zuApvWdpQdBHM/swxP962s8HIxPO3alBH5t3m/dl+f4CMUug6sJb7Pww8d13/9WSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash._baseuniq": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz", + "integrity": "sha512-Ja1YevpHZctlI5beLA7oc5KNDhGcPixFhcqSiORHNsp/1QTv7amAXzw+gu4YOvErqVlMVyIJGgtzeepCnnur0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash._createset": "~4.0.0", + "lodash._root": "~3.0.0" + } + }, + "node_modules/lodash._createset": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/lodash._createset/-/lodash._createset-4.0.3.tgz", + "integrity": "sha512-GTkC6YMprrJZCYU3zcqZj+jkXkrXzq3IPBcF/fIPpNEAB4hZEtXU8zp/RwKOvZl43NUmwDbyRk3+ZTbeRdEBXA==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash._root": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash._root/-/lodash._root-3.0.1.tgz", + "integrity": "sha512-O0pWuFSK6x4EXhM1dhZ8gchNtG7JMqBtrHdoUFUWXD7dJnNSUze1GuyQr5sOs0aCvgGeI3o/OJW8f4ca7FDxmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash._stringtopath": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/lodash._stringtopath/-/lodash._stringtopath-4.8.0.tgz", + "integrity": "sha512-SXL66C731p0xPDC5LZg4wI5H+dJo/EO4KTqOMwLYCH3+FmmfAKJEZCm6ohGpI+T1xwsDsJCfL4OnhorllvlTPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash._basetostring": "~4.12.0" + } + }, + "node_modules/lodash.throttle": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", + "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==", + "license": "MIT" + }, + "node_modules/lodash.uniqby": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.5.0.tgz", + "integrity": "sha512-IRt7cfTtHy6f1aRVA5n7kT8rgN3N1nH6MOWLcHfpWG2SH19E3JksLK38MktLxZDhlAjCP9jpIXkOnRXlu6oByQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash._baseiteratee": "~4.7.0", + "lodash._baseuniq": "~4.6.0" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/negotiator/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/qs": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "optional": true, + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/set-cookie-parser": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", + "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", + "license": "MIT" + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/srvx": { + "version": "0.11.22", + "resolved": "https://registry.npmjs.org/srvx/-/srvx-0.11.22.tgz", + "integrity": "sha512-LqZxxBDMKuMAZzFzJnDCkFOrs9MZQZr0LvHiO/SuSZVdQaXD7xQ5UWTUxheJrQPve1qk9MG2B/yttUvJxw8egQ==", + "license": "MIT", + "bin": { + "srvx": "bin/srvx.mjs" + }, + "engines": { + "node": ">=20.16.0" + } + }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT", + "optional": true + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tus-js-client": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/tus-js-client/-/tus-js-client-4.3.1.tgz", + "integrity": "sha512-ZLeYmjrkaU1fUsKbIi8JML52uAocjEZtBx4DKjRrqzrZa0O4MYwT6db+oqePlspV+FxXJAyFBc/L5gwUi2OFsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.1.2", + "combine-errors": "^3.0.3", + "is-stream": "^2.0.0", + "js-base64": "^3.7.2", + "lodash.throttle": "^4.1.1", + "proper-lockfile": "^4.1.2", + "url-parse": "^1.5.7" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + } + } +} diff --git a/storage-api/package.json b/storage-api/package.json new file mode 100644 index 0000000..a6a4599 --- /dev/null +++ b/storage-api/package.json @@ -0,0 +1,24 @@ +{ + "name": "neurojson-storage-api", + "version": "0.1.0", + "private": true, + "description": "Zodiac storage service for NeuroJSON raw uploads (tus).", + "type": "module", + "engines": { + "node": ">=20.19.0" + }, + "scripts": { + "start": "node src/server.js", + "dev": "node --watch src/server.js", + "self-test": "node scripts/self-test.js" + }, + "dependencies": { + "@tus/file-store": "^2.1.1", + "@tus/server": "^2.4.5", + "dotenv": "^17.2.3", + "express": "^5.1.0" + }, + "devDependencies": { + "tus-js-client": "^4.3.1" + } +} diff --git a/storage-api/scripts/call-internal.js b/storage-api/scripts/call-internal.js new file mode 100644 index 0000000..d8434ce --- /dev/null +++ b/storage-api/scripts/call-internal.js @@ -0,0 +1,22 @@ +// Send a signed request to the storage API's /internal endpoints, as REN would. +// Usage: node scripts/call-internal.js [''] +// e.g. node scripts/call-internal.js GET /status +// node scripts/call-internal.js GET /uploads/ +// node scripts/call-internal.js POST /uploads//decision '{"decision":"commit","rawId":""}' +// node scripts/call-internal.js POST /uploads//ack '{"action":"keep"}' +import { config } from "../src/config.js"; +import { signRequest } from "../src/lib/storageSignature.js"; + +const [method = "GET", sub = "/status", json = ""] = process.argv.slice(2); +const path = `${config.basePath}/internal${sub}`; +const url = `http://${config.host}:${process.env.CALL_PORT || config.port}${path}`; +const headers = { + ...(json ? { "Content-Type": "application/json" } : {}), + ...signRequest( + { method, path, body: json }, + config.hmacKeys, + config.hmacActiveKid + ), +}; +const res = await fetch(url, { method, headers, body: json || undefined }); +console.log(res.status, await res.text()); diff --git a/storage-api/scripts/self-test.js b/storage-api/scripts/self-test.js new file mode 100644 index 0000000..0b5afee --- /dev/null +++ b/storage-api/scripts/self-test.js @@ -0,0 +1,113 @@ +// Self-test for the storage API's token + HMAC helpers and config (prints no +// secrets). Usage: npm run self-test (reads storage-api/.env) +import crypto from "node:crypto"; +import { config } from "../src/config.js"; // also validates .env (exits if wrong) +import { verifyToken } from "../src/lib/uploadTokens.js"; +import { signRequest, verifyRequest } from "../src/lib/storageSignature.js"; + +let failed = 0; +const check = (name, fn) => { + try { + fn(); + console.log("PASS", name); + } catch (e) { + failed++; + console.log("FAIL", name, "-", e.message); + } +}; +const expectThrow = (fn, msg) => { + try { + fn(); + } catch { + return; + } + throw new Error(`expected failure: ${msg}`); +}; + +// --- Tokens: sign with a THROWAWAY key in the same format REN uses --------- +const { privateKey, publicKey } = crypto.generateKeyPairSync("ed25519"); +const pub = { "test-kid": publicKey }; +const b64 = (v) => Buffer.from(JSON.stringify(v)).toString("base64url"); +const makeToken = (claims, ttl = 900) => { + const iat = Math.floor(Date.now() / 1000); + const header = { alg: "EdDSA", typ: "JWT", kid: "test-kid" }; + const payload = { + iss: "neurojson-api", + aud: "zodiac-storage", + typ: "tus-upload", + iat, + exp: iat + ttl, + ...claims, + }; + const input = `${b64(header)}.${b64(payload)}`; + const sig = crypto.sign(null, Buffer.from(input), privateKey); + return `${input}.${sig.toString("base64url")}`; +}; +const token = makeToken({ uid: crypto.randomUUID(), len: 1234 }); + +check("valid token verifies", () => { + if (verifyToken(token, pub, "tus-upload").len !== 1234) { + throw new Error("len claim wrong"); + } +}); +check("tampered payload is rejected", () => { + const [h, p, s] = token.split("."); + const bad = JSON.parse(Buffer.from(p, "base64url").toString()); + bad.len = 999999; + expectThrow( + () => verifyToken(`${h}.${b64(bad)}.${s}`, pub, "tus-upload"), + "tamper" + ); +}); +check("wrong token type is rejected", () => + expectThrow(() => verifyToken(token, pub, "file-read"), "typ") +); +check("unknown kid is rejected", () => + expectThrow(() => verifyToken(token, {}, "tus-upload"), "kid") +); +check("expired token is rejected", () => + expectThrow( + () => verifyToken(makeToken({}, -120), pub, "tus-upload"), + "exp" + ) +); + +// --- HMAC: uses the CONFIGURED Zodiac secret -------------------------------- +const body = JSON.stringify({ event: "verified" }); +const path = "/neurojson-storage/internal/uploads/x"; +const headers = signRequest( + { method: "POST", path, body }, + config.hmacKeys, + config.hmacActiveKid +); +check("valid HMAC request verifies", () => { + const r = verifyRequest( + { method: "POST", path, headers, rawBody: body }, + config.hmacKeys + ); + if (!r.ok) throw new Error(r.reason); +}); +check("HMAC rejects changed body", () => { + const r = verifyRequest( + { method: "POST", path, headers, rawBody: body + " " }, + config.hmacKeys + ); + if (r.ok) throw new Error("accepted"); +}); +check("HMAC rejects changed path", () => { + const r = verifyRequest( + { method: "POST", path: path + "?x=1", headers, rawBody: body }, + config.hmacKeys + ); + if (r.ok) throw new Error("accepted"); +}); + +// --- Config ---------------------------------------------------------------- +check("configured public keys load", () => { + const kids = Object.keys(config.tokenPublicKeys); + if (!kids.length) throw new Error("none"); + console.log(" kids:", kids.join(", ")); +}); + +console.log(failed ? `\n${failed} check(s) FAILED` : "\nAll checks passed"); +process.exit(failed ? 1 : 0); diff --git a/storage-api/scripts/test-upload.js b/storage-api/scripts/test-upload.js new file mode 100644 index 0000000..a8f2c34 --- /dev/null +++ b/storage-api/scripts/test-upload.js @@ -0,0 +1,72 @@ +// Upload a test file to the storage API with tus-js-client (Node), like the +// browser will. The test file starts with "PK\x03\x04" so the later ZIP check +// passes; it's saved under storage-dev/test-files/ so a resumed run sends the +// exact same bytes. +// +// Usage: +// node scripts/test-upload.js '' \ +// [--stop-after-mb N] [--resume] [--chunk-mb 5] [--endpoint URL] +// --stop-after-mb N abort once N MB are uploaded (to test resuming) +// --resume continue an existing upload (HEAD → PATCH from its offset) +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import * as tus from "tus-js-client"; + +const args = process.argv.slice(2); +const opt = (name, def) => { + const i = args.indexOf(name); + return i === -1 ? def : args[i + 1]; +}; +let info; +try { + info = JSON.parse(args[0]); +} catch { + console.log("first argument must be the JSON printed by mint-test-upload-token.js"); + process.exit(1); +} +const endpoint = opt( + "--endpoint", + "http://127.0.0.1:8090/neurojson-storage/files" +); +const chunkSize = Number(opt("--chunk-mb", 5)) * 1024 * 1024; +const stopAfter = opt("--stop-after-mb") && Number(opt("--stop-after-mb")) * 1024 * 1024; +const resume = args.includes("--resume"); + +// Same bytes every run for the same upload id. +const dir = path.resolve("storage-dev/test-files"); +fs.mkdirSync(dir, { recursive: true }); +const file = path.join(dir, `${info.uploadId}.zip`); +if (!fs.existsSync(file)) { + const buf = crypto.randomBytes(info.size); + Buffer.from([0x50, 0x4b, 0x03, 0x04]).copy(buf, 0); // "PK\x03\x04" + fs.writeFileSync(file, buf); +} +const data = fs.readFileSync(file); + +let stopped = false; +const upload = new tus.Upload(data, { + endpoint, + uploadUrl: resume ? `${endpoint}/${info.uploadId}` : undefined, + uploadSize: data.length, + chunkSize, + headers: { Authorization: `Bearer ${info.token}` }, + metadata: { filename: `${info.uploadId}.zip` }, + retryDelays: [0, 1000], + onProgress: (sent, total) => { + process.stdout.write(`\r${(sent / 1048576).toFixed(1)} / ${(total / 1048576).toFixed(1)} MB`); + if (stopAfter && sent >= stopAfter && !stopped) { + stopped = true; + upload.abort().then(() => + console.log(`\nstopped at ${(sent / 1048576).toFixed(1)} MB (run again with --resume)`) + ); + } + }, + onError: (err) => { + const r = err.originalResponse; + console.log(`\nFAILED: ${r ? `HTTP ${r.getStatus()} ${r.getBody().trim()}` : err.message}`); + process.exit(1); + }, + onSuccess: () => console.log(`\nDONE: ${upload.url}`), +}); +upload.start(); diff --git a/storage-api/src/config.js b/storage-api/src/config.js new file mode 100644 index 0000000..c028479 --- /dev/null +++ b/storage-api/src/config.js @@ -0,0 +1,101 @@ +// Read and check all settings at startup. If anything is wrong, list every +// problem and refuse to start (better than failing on the first upload). +import "dotenv/config"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +const problems = []; +const env = process.env; + +const parseJson = (name) => { + try { + return JSON.parse(env[name] || ""); + } catch { + problems.push(`${name} must be valid JSON`); + return {}; + } +}; + +const basePath = (env.BASE_PATH || "/neurojson-storage").replace(/\/+$/, ""); +if (!basePath.startsWith("/")) problems.push("BASE_PATH must start with /"); + +if (!env.UPLOAD_ROOT) problems.push("UPLOAD_ROOT is required"); +const uploadRoot = path.resolve(env.UPLOAD_ROOT || "."); + +const allowedOrigins = (env.ALLOWED_ORIGINS || "") + .split(",") + .map((s) => s.trim()) + .filter(Boolean); +if (!allowedOrigins.length) problems.push("ALLOWED_ORIGINS is required"); + +// Public keys by kid; each must be a real Ed25519 public key. +const tokenPublicKeys = {}; +for (const [kid, pem] of Object.entries( + parseJson("UPLOAD_TOKEN_PUBLIC_KEYS") +)) { + try { + const key = crypto.createPublicKey(pem); + if (key.asymmetricKeyType !== "ed25519") throw new Error(); + tokenPublicKeys[kid] = key; + } catch { + problems.push( + `UPLOAD_TOKEN_PUBLIC_KEYS["${kid}"] is not an Ed25519 public key` + ); + } +} +if (!Object.keys(tokenPublicKeys).length) { + problems.push("UPLOAD_TOKEN_PUBLIC_KEYS has no keys"); +} + +const hmacKeys = parseJson("STORAGE_HMAC_KEYS"); +const hmacActiveKid = env.STORAGE_HMAC_ACTIVE_KID; +if (!hmacActiveKid || !hmacKeys[hmacActiveKid]) { + problems.push( + "STORAGE_HMAC_ACTIVE_KID must name a key in STORAGE_HMAC_KEYS" + ); +} + +const num = (name, def) => { + const v = Number(env[name] ?? def); + if (!Number.isFinite(v) || v < 0) { + problems.push(`${name} must be a positive number`); + } + return v; +}; + +export const config = { + host: env.HOST || "127.0.0.1", + port: num("PORT", 8090), + basePath, + uploadRoot, + incomingDir: path.join(uploadRoot, ".incoming"), + outboxDir: path.join(uploadRoot, ".outbox"), + allowedOrigins, + mainApiUrl: (env.MAIN_API_URL || "").replace(/\/+$/, ""), // "" = pushes off + tokenPublicKeys, + hmacKeys, + hmacActiveKid, + maxUploadBytes: num("MAX_UPLOAD_BYTES", 50000000000), // 50 GB (decimal) + diskReservePercent: num("DISK_RESERVE_PERCENT", 5), +}; + +if (problems.length) { + console.error( + "storage-api: invalid configuration:\n - " + problems.join("\n - ") + ); + process.exit(1); +} + +// Working folders (the root itself must already exist and be writable). +try { + fs.accessSync(uploadRoot, fs.constants.W_OK); + for (const dir of [config.incomingDir, config.outboxDir]) { + fs.mkdirSync(dir, { recursive: true }); + } +} catch (e) { + console.error( + `storage-api: UPLOAD_ROOT is not writable: ${uploadRoot} (${e.code})` + ); + process.exit(1); +} diff --git a/storage-api/src/lib/diskSpace.js b/storage-api/src/lib/diskSpace.js new file mode 100644 index 0000000..12424c5 --- /dev/null +++ b/storage-api/src/lib/diskSpace.js @@ -0,0 +1,34 @@ +// Free space minus what in-progress uploads still need, so two big uploads +// can't each pass the check and then fill the disk together. Computed fresh +// from .incoming/ (survives restarts; creates are rare, so a scan is cheap). +import fs from "node:fs/promises"; +import path from "node:path"; +import { config } from "../config.js"; + +// Bytes other active uploads still have to write. +const reservedBytes = async (exceptId) => { + let total = 0; + for (const name of await fs.readdir(config.incomingDir)) { + if (!name.endsWith(".json")) continue; + const id = name.slice(0, -5); + if (id === exceptId) continue; + try { + const meta = JSON.parse( + await fs.readFile(path.join(config.incomingDir, name), "utf8") + ); + const written = (await fs.stat(path.join(config.incomingDir, id))).size; + if (Number.isFinite(meta.size)) total += Math.max(0, meta.size - written); + } catch { + // half-written or removed meanwhile — ignore + } + } + return total; +}; + +// true if `size` more bytes fit, keeping DISK_RESERVE_PERCENT of the disk free. +export const hasRoomFor = async (size, uploadId) => { + const s = await fs.statfs(config.uploadRoot); + const free = s.bavail * s.bsize; + const margin = (s.blocks * s.bsize * config.diskReservePercent) / 100; + return free - (await reservedBytes(uploadId)) - margin >= size; +}; diff --git a/storage-api/src/lib/fileChecks.js b/storage-api/src/lib/fileChecks.js new file mode 100644 index 0000000..311a78f --- /dev/null +++ b/storage-api/src/lib/fileChecks.js @@ -0,0 +1,38 @@ +// File checks for a finished upload. The ZIP check is SCREENING ONLY (first +// bytes) — the archive is never opened or extracted here. +import crypto from "node:crypto"; +import fs from "node:fs"; +import fsp from "node:fs/promises"; + +const ZIP_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04]); // "PK\x03\x04" + +export const looksLikeZip = async (file) => { + const fh = await fsp.open(file, "r"); + try { + const buf = Buffer.alloc(4); + const { bytesRead } = await fh.read(buf, 0, 4, 0); + return bytesRead === 4 && buf.equals(ZIP_MAGIC); + } finally { + await fh.close(); + } +}; + +// Streamed (constant memory), so it works for very large files. +export const sha256File = (file) => + new Promise((resolve, reject) => { + const hash = crypto.createHash("sha256"); + fs.createReadStream(file, { highWaterMark: 4 * 1024 * 1024 }) + .on("data", (chunk) => hash.update(chunk)) + .on("error", reject) + .on("end", () => resolve(hash.digest("hex"))); + }); + +// Make a rename durable: fsync the directory that now holds the file. +export const fsyncDir = async (dir) => { + const fh = await fsp.open(dir, "r"); + try { + await fh.sync(); + } finally { + await fh.close(); + } +}; diff --git a/storage-api/src/lib/mainApiClient.js b/storage-api/src/lib/mainApiClient.js new file mode 100644 index 0000000..fa26340 --- /dev/null +++ b/storage-api/src/lib/mainApiClient.js @@ -0,0 +1,50 @@ +// Signed calls from Zodiac to REN. Classifies the answer for the outbox: +// ok → 2xx with a JSON body +// retry → network error / timeout / 5xx (try again later) +// permanent → 4xx (retrying won't help: wrong keys, unknown upload, …) +import { config } from "../config.js"; +import { signRequest } from "./storageSignature.js"; + +export const pushEnabled = () => Boolean(config.mainApiUrl); + +export const sendUploadEvent = async (uploadId, payload) => { + const url = new URL( + `${config.mainApiUrl}/internal/storage/uploads/${uploadId}/events` + ); + const body = JSON.stringify(payload); + const headers = { + "Content-Type": "application/json", + // REN verifies against its req.originalUrl, i.e. this exact path + query. + ...signRequest( + { method: "POST", path: url.pathname + url.search, body }, + config.hmacKeys, + config.hmacActiveKid + ), + }; + let res; + try { + res = await fetch(url, { + method: "POST", + headers, + body, + signal: AbortSignal.timeout(30_000), + }); + } catch (e) { + return { kind: "retry", error: `network: ${e.message}` }; + } + const text = await res.text(); + if (res.ok) { + try { + return { kind: "ok", body: JSON.parse(text) }; + } catch { + return { kind: "retry", error: "REN sent a non-JSON reply" }; + } + } + if (res.status >= 500) { + return { kind: "retry", error: `REN HTTP ${res.status}` }; + } + return { + kind: "permanent", + error: `REN HTTP ${res.status}: ${text.slice(0, 200)}`, + }; +}; diff --git a/storage-api/src/lib/paths.js b/storage-api/src/lib/paths.js new file mode 100644 index 0000000..1a45137 --- /dev/null +++ b/storage-api/src/lib/paths.js @@ -0,0 +1,20 @@ +// All storage paths are built ONLY from token claims (UUIDs), never from +// anything else the client sends, and must stay inside UPLOAD_ROOT. +import path from "node:path"; +import { config } from "../config.js"; + +const inside = (p) => { + const root = config.uploadRoot + path.sep; + if (!p.startsWith(root)) throw new Error("path escapes UPLOAD_ROOT"); + return p; +}; + +export const incomingFile = (uploadId) => + inside(path.join(config.incomingDir, uploadId)); + +export const rawObjectDir = (internalId, rawId) => + inside(path.join(config.uploadRoot, internalId, "raw", rawId)); + +// Relative key stored in Postgres (never an absolute path). +export const rawZipKey = (internalId, rawId) => + `${internalId}/raw/${rawId}/source.zip`; diff --git a/storage-api/src/lib/storageSignature.js b/storage-api/src/lib/storageSignature.js new file mode 100644 index 0000000..74f7e43 --- /dev/null +++ b/storage-api/src/lib/storageSignature.js @@ -0,0 +1,59 @@ +// HMAC-SHA256 signatures for server-to-server calls between REN and Zodiac +// (both directions). Same scheme as backend/src/lib/storageSignature.js — +// keep the two in sync. Signs timestamp + method + path(+query) + SHA-256 of +// the raw body; secrets are a kid -> secret map so they can rotate. +import crypto from "node:crypto"; + +const MAX_SKEW_SECONDS = 300; +export const HEADER_NAMES = { + H_KID: "x-nj-key-id", + H_TS: "x-nj-timestamp", + H_SIG: "x-nj-signature", +}; +const { H_KID, H_TS, H_SIG } = HEADER_NAMES; + +const sha256hex = (buf) => + crypto.createHash("sha256").update(buf || "").digest("hex"); +const canonical = (ts, method, path, body) => + [ts, String(method).toUpperCase(), path, sha256hex(body)].join("\n"); + +// Headers to attach to an outgoing request. `body` = the exact bytes/string sent. +export const signRequest = ({ method, path, body = "" }, keys, activeKid) => { + const secret = keys[activeKid]; + if (!activeKid || !secret) throw new Error("Storage HMAC key not configured."); + const ts = String(Math.floor(Date.now() / 1000)); + const sig = crypto + .createHmac("sha256", secret) + .update(canonical(ts, method, path, body)) + .digest("base64url"); + return { [H_KID]: activeKid, [H_TS]: ts, [H_SIG]: sig }; +}; + +// Check an incoming request. `path` must include the query string exactly as +// sent; `rawBody` must be the unparsed bytes. Returns { ok, reason }. +export const verifyRequest = ({ method, path, headers, rawBody }, keys) => { + const kid = headers[H_KID]; + const ts = headers[H_TS]; + const sig = headers[H_SIG]; + if (!kid || !ts || !sig) { + return { ok: false, reason: "missing signature headers" }; + } + const secret = keys[kid]; + if (!secret) return { ok: false, reason: "unknown key id" }; + const now = Math.floor(Date.now() / 1000); + if (!/^\d+$/.test(ts) || Math.abs(now - Number(ts)) > MAX_SKEW_SECONDS) { + return { ok: false, reason: "timestamp out of range" }; + } + const expected = crypto + .createHmac("sha256", secret) + .update(canonical(ts, method, path, rawBody)) + .digest(); + const given = Buffer.from(String(sig), "base64url"); + if ( + given.length !== expected.length || + !crypto.timingSafeEqual(given, expected) + ) { + return { ok: false, reason: "bad signature" }; + } + return { ok: true, kid }; +}; diff --git a/storage-api/src/lib/tusAuth.js b/storage-api/src/lib/tusAuth.js new file mode 100644 index 0000000..a8b2bfb --- /dev/null +++ b/storage-api/src/lib/tusAuth.js @@ -0,0 +1,40 @@ +// Upload-token checks for tus requests. Every tus request (create, chunk, +// resume check, cancel) must carry `Authorization: Bearer `. Verified +// once per request and cached (tus calls several hooks with the same Request). +import { config } from "../config.js"; +import { verifyToken } from "./uploadTokens.js"; + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const cache = new WeakMap(); + +// tus sends { status_code, body } to the client; body must not leak internals. +export const tusError = (status_code, message) => ({ + status_code, + body: `${message}\n`, +}); + +export const getClaims = (req) => { + if (cache.has(req)) return cache.get(req); + const auth = req.headers.get("authorization") || ""; + const token = auth.startsWith("Bearer ") ? auth.slice(7) : ""; + let claims; + try { + claims = verifyToken(token, config.tokenPublicKeys, "tus-upload"); + } catch { + throw tusError(401, "Invalid or expired upload token."); + } + // Paths are built from these claims, so check their shape strictly. + if ( + !UUID_RE.test(String(claims.uid)) || + !UUID_RE.test(String(claims.iid)) || + !UUID_RE.test(String(claims.sid)) || + !Number.isSafeInteger(claims.len) || + claims.len <= 0 || + claims.len > config.maxUploadBytes + ) { + throw tusError(401, "Invalid upload token."); + } + cache.set(req, claims); + return claims; +}; diff --git a/storage-api/src/lib/uploadTokens.js b/storage-api/src/lib/uploadTokens.js new file mode 100644 index 0000000..f755e52 --- /dev/null +++ b/storage-api/src/lib/uploadTokens.js @@ -0,0 +1,50 @@ +// Verify the short-lived EdDSA (Ed25519) tokens REN issues for uploads. +// Same strict checks as backend/src/lib/uploadTokens.js verifyToken — keep the +// two in sync (scripts/self-test.js exercises this copy). Zodiac never signs. +import crypto from "node:crypto"; + +const ISSUER = "neurojson-api"; +const AUDIENCE = "zodiac-storage"; +const CLOCK_LEEWAY_SECONDS = 30; + +// publicKeys: { [kid]: KeyObject | pem }. Throws on any problem; returns claims. +export const verifyToken = (token, publicKeys, expectedTyp) => { + const parts = String(token || "").split("."); + if (parts.length !== 3) throw new Error("malformed token"); + const [h, p, s] = parts; + const header = JSON.parse(Buffer.from(h, "base64url").toString()); + if (header.alg !== "EdDSA") throw new Error("bad alg"); // never trust other algs + const keyMaterial = publicKeys[header.kid]; + if (!keyMaterial) throw new Error("unknown kid"); + const key = + typeof keyMaterial === "string" + ? crypto.createPublicKey(keyMaterial) + : keyMaterial; + if (key.asymmetricKeyType !== "ed25519") throw new Error("bad key type"); + const ok = crypto.verify( + null, + Buffer.from(`${h}.${p}`), + key, + Buffer.from(s, "base64url") + ); + if (!ok) throw new Error("bad signature"); + const claims = JSON.parse(Buffer.from(p, "base64url").toString()); + const now = Math.floor(Date.now() / 1000); + if (claims.iss !== ISSUER || claims.aud !== AUDIENCE) { + throw new Error("bad issuer/audience"); + } + if (claims.typ !== expectedTyp) throw new Error("wrong token type"); + if ( + typeof claims.exp !== "number" || + claims.exp + CLOCK_LEEWAY_SECONDS < now + ) { + throw new Error("expired"); + } + if ( + typeof claims.iat !== "number" || + claims.iat - CLOCK_LEEWAY_SECONDS > now + ) { + throw new Error("issued in the future"); + } + return claims; +}; diff --git a/storage-api/src/outbox/outbox.js b/storage-api/src/outbox/outbox.js new file mode 100644 index 0000000..0b4cae6 --- /dev/null +++ b/storage-api/src/outbox/outbox.js @@ -0,0 +1,105 @@ +// Crash-safe job records on disk, one per finished upload: +// .outbox/.json pending +// .outbox/done/.json finished (kept for status lookups; cleaned up later) +// .outbox/failed/.json parked for an admin (REN said 4xx, or files missing) +import fs from "node:fs/promises"; +import path from "node:path"; +import { config } from "../config.js"; + +const DIR = config.outboxDir; +const DONE = path.join(DIR, "done"); +const FAILED = path.join(DIR, "failed"); +await fs.mkdir(DONE, { recursive: true }); +await fs.mkdir(FAILED, { recursive: true }); + +const fileIn = (dir, id) => path.join(dir, `${id}.json`); + +// One change at a time per upload (worker vs. REN's /internal calls). +// In-memory is enough: the service runs as a single process. +const locks = new Map(); +export const withLock = async (id, fn) => { + while (locks.has(id)) await locks.get(id); + let release; + locks.set( + id, + new Promise((r) => { + release = r; + }) + ); + try { + return await fn(); + } finally { + locks.delete(id); + release(); + } +}; + +// temp file → fsync → rename, so a crash never leaves a half-written record. +const writeAtomic = async (file, obj) => { + const tmp = `${file}.tmp`; + const fh = await fs.open(tmp, "w"); + try { + await fh.writeFile(JSON.stringify(obj, null, 2)); + await fh.sync(); + } finally { + await fh.close(); + } + await fs.rename(tmp, file); +}; + +// Look in pending, done and failed (also used by the status endpoint). +export const findRecord = async (id) => { + for (const [where, dir] of [ + ["pending", DIR], + ["done", DONE], + ["failed", FAILED], + ]) { + try { + return { + where, + rec: JSON.parse(await fs.readFile(fileIn(dir, id), "utf8")), + }; + } catch { + // not here + } + } + return null; +}; + +// Returns false if a record for this upload already exists (never overwrite). +export const createRecord = async (rec) => { + if (await findRecord(rec.uploadId)) return false; + const now = new Date().toISOString(); + await writeAtomic(fileIn(DIR, rec.uploadId), { + ...rec, + attempts: 0, + nextAttemptAt: now, + lastError: null, + createdAt: now, + updatedAt: now, + }); + return true; +}; + +export const saveRecord = (rec) => + writeAtomic(fileIn(DIR, rec.uploadId), { + ...rec, + updatedAt: new Date().toISOString(), + }); + +export const listPendingIds = async () => + (await fs.readdir(DIR)) + .filter((n) => n.endsWith(".json")) + .map((n) => n.slice(0, -5)); + +export const readPending = async (id) => + JSON.parse(await fs.readFile(fileIn(DIR, id), "utf8")); + +// Final state: save, then move out of the pending folder. +export const closeRecord = async (rec, outcome) => { + await saveRecord(rec); + await fs.rename( + fileIn(DIR, rec.uploadId), + fileIn(outcome === "failed" ? FAILED : DONE, rec.uploadId) + ); +}; diff --git a/storage-api/src/outbox/worker.js b/storage-api/src/outbox/worker.js new file mode 100644 index 0000000..8cade43 --- /dev/null +++ b/storage-api/src/outbox/worker.js @@ -0,0 +1,278 @@ +// Drives each finished upload to completion, one record at a time (hashing +// large files is disk-heavy). Every state change is saved before the next +// step, so after a crash or restart the worker continues where it stopped. +// +// verifying ─► awaiting_decision ─[verified]─► committing ─► committed ─[committed]─► done +// │ └─► discarding ─► done (discard on committed → discarding_committed) +// └─► rejecting ─[failed]─► done +// +// Pull mode (MAIN_API_URL empty): the file work still runs, but the worker +// stops at the states that need REN's answer; REN drives those via /internal. +import fs from "node:fs/promises"; +import path from "node:path"; +import { + listPendingIds, + readPending, + saveRecord, + closeRecord, + withLock, +} from "./outbox.js"; +import { incomingFile, rawObjectDir, rawZipKey } from "../lib/paths.js"; +import { looksLikeZip, sha256File, fsyncDir } from "../lib/fileChecks.js"; +import { pushEnabled, sendUploadEvent } from "../lib/mainApiClient.js"; + +const TICK_MS = 5000; +const BACKOFF_SECONDS = [5, 30, 120, 600, 1800, 3600]; +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +// States where the next move needs REN's answer (pull mode waits here). +export const WAITS_FOR_REN = new Set([ + "awaiting_decision", + "rejecting", + "committed", +]); + +const exists = (p) => + fs.access(p).then( + () => true, + () => false + ); +const removeIncoming = async (id) => { + await fs.rm(incomingFile(id), { force: true }); + await fs.rm(`${incomingFile(id)}.json`, { force: true }); +}; + +// A problem retrying won't fix (bad reply, files missing) → park the record. +export class Permanent extends Error {} + +const ready = (rec) => { + rec.attempts = 0; + rec.lastError = null; + rec.nextAttemptAt = new Date().toISOString(); +}; + +const retryLater = async (rec, error) => { + const delay = + BACKOFF_SECONDS[Math.min(rec.attempts, BACKOFF_SECONDS.length - 1)]; + rec.attempts += 1; + rec.lastError = error; + rec.nextAttemptAt = new Date(Date.now() + delay * 1000).toISOString(); + await saveRecord(rec); + console.warn( + `outbox ${rec.uploadId}: ${rec.state} retry #${rec.attempts} in ${delay}s (${error})` + ); +}; + +const park = async (rec, error) => { + rec.lastError = error; + rec.parkedFrom = rec.state; + rec.state = "failed"; + await closeRecord(rec, "failed"); + console.error( + `outbox ${rec.uploadId}: PARKED in .outbox/failed/ (${error})` + ); +}; + +const finish = async (rec) => { + rec.state = "done"; + await closeRecord(rec, "done"); + console.log(`outbox ${rec.uploadId}: done`); +}; + +// Ask REN; returns the reply body, or throws Permanent / a retryable Error. +const askRen = async (rec, payload) => { + const r = await sendUploadEvent(rec.uploadId, payload); + if (r.kind === "ok") return r.body; + if (r.kind === "permanent") throw new Permanent(r.error); + throw new Error(r.error); +}; + +const setState = async (rec, state, extra = {}) => { + Object.assign(rec, extra, { state }); + ready(rec); + await saveRecord(rec); + console.log(`outbox ${rec.uploadId}: → ${state}`); +}; + +const STEPS = { + // Size, ZIP signature, SHA-256 of the file sitting in .incoming/. + async verifying(rec) { + const file = incomingFile(rec.uploadId); + let size; + try { + size = (await fs.stat(file)).size; + } catch { + throw new Permanent("incoming file is missing"); + } + let reason = null; + if (size !== rec.expectedSize) reason = "size_mismatch"; + else if (!(await looksLikeZip(file))) reason = "not_zip"; + if (reason) return setState(rec, "rejecting", { reason }); + const sha256 = await sha256File(file); + await setState(rec, "awaiting_decision", { size, sha256 }); + }, + + // Tell REN the file failed the checks, then delete it. + async rejecting(rec) { + await askRen(rec, { event: "failed", reason: rec.reason }); + await applyAck(rec); + }, + + // "verified" → REN answers commit (with rawId) or discard. + async awaiting_decision(rec) { + const reply = await askRen(rec, { + event: "verified", + size: rec.size, + sha256: rec.sha256, + }); + await applyDecision(rec, reply); + }, + + // Move .incoming/ → /raw//source.zip (atomic rename). + async committing(rec) { + const dir = rawObjectDir(rec.internalId, rec.rawId); + const src = incomingFile(rec.uploadId); + const dest = path.join(dir, "source.zip"); + await fs.mkdir(dir, { recursive: true }); + const haveSrc = await exists(src); + const haveDest = await exists(dest); + if (haveSrc && haveDest) { + throw new Permanent("both incoming and final file exist"); + } + if (!haveSrc && !haveDest) { + throw new Permanent("file is missing (neither incoming nor final)"); + } + if (haveSrc) await fs.rename(src, dest); // else: moved before a crash — continue + await fsyncDir(dir); + if ((await fs.stat(dest)).size !== rec.size) { + throw new Permanent("final file size changed"); + } + await fs.rm(`${src}.json`, { force: true }); // tus metadata no longer needed + await setState(rec, "committed", { + storageKey: rawZipKey(rec.internalId, rec.rawId), + }); + }, + + // "committed" → REN records the object; keep, or discard if cancelled meanwhile. + async committed(rec) { + const reply = await askRen(rec, { + event: "committed", + size: rec.size, + sha256: rec.sha256, + }); + await applyAck(rec, reply); + }, + + async discarding(rec) { + await removeIncoming(rec.uploadId); + await finish(rec); + }, + + async discarding_committed(rec) { + await fs.rm(rawObjectDir(rec.internalId, rec.rawId), { + recursive: true, + force: true, + }); + await finish(rec); + }, +}; + +// Apply REN's commit/discard answer (used by push mode here and by the pull +// endpoint later). Validates the rawId so a bad reply can't pick a path. +export const applyDecision = async (rec, reply) => { + if (reply.decision === "commit") { + if (!UUID_RE.test(String(reply.rawId))) { + throw new Permanent("REN sent an invalid rawId"); + } + const key = rawZipKey(rec.internalId, reply.rawId); + if (reply.storageKey && reply.storageKey !== key) { + throw new Permanent("REN storageKey does not match"); + } + return setState(rec, "committing", { rawId: reply.rawId }); + } + if (reply.decision === "discard") { + return setState(rec, "discarding", { reason: reply.reason || null }); + } + throw new Permanent("REN sent an unknown decision"); +}; + +// REN's acknowledgement of a final result (push reply or pull /ack). +export const applyAck = async (rec, reply = {}) => { + if (rec.state === "committed") { + if (reply.action === "discard") { + return setState(rec, "discarding_committed", { + reason: reply.reason || null, + }); + } + if (reply.action === "keep") return finish(rec); + throw new Permanent( + "acknowledging a committed upload needs action keep or discard" + ); + } + if (rec.state === "rejecting") { + await removeIncoming(rec.uploadId); + return finish(rec); + } + throw new Permanent(`nothing to acknowledge in state ${rec.state}`); +}; + +// Run one record forward until it waits, retries, or finishes. +const advance = async (rec) => { + for (let i = 0; i < 10; i++) { + const step = STEPS[rec.state]; + if (!step) return; + if (!pushEnabled() && WAITS_FOR_REN.has(rec.state)) return; // pull mode + const before = rec.state; + try { + await step(rec); + } catch (e) { + if (e instanceof Permanent) await park(rec, e.message); + else await retryLater(rec, e.message); + return; + } + if (rec.state === before || rec.state === "done") return; + } +}; + +let running = false; +let again = false; +let timer = null; + +const tick = async () => { + if (running) { + again = true; + return; + } + running = true; + try { + for (const id of await listPendingIds()) { + // Read fresh inside the lock, so REN's /internal calls and the worker + // never work on the same record at once (or on a stale copy). + await withLock(id, async () => { + let rec; + try { + rec = await readPending(id); + } catch { + return; // finished or removed meanwhile + } + if (new Date(rec.nextAttemptAt) > new Date()) return; + await advance(rec); + }); + } + } catch (e) { + console.error("outbox worker:", e.message); + } finally { + running = false; + if (again) { + again = false; + setImmediate(tick); + } + } +}; + +export const wakeWorker = () => setImmediate(tick); +export const startWorker = () => { + timer = setInterval(tick, TICK_MS); + wakeWorker(); // pick up records left from before a restart +}; +export const stopWorker = () => clearInterval(timer); diff --git a/storage-api/src/routes/internal.js b/storage-api/src/routes/internal.js new file mode 100644 index 0000000..c88ee83 --- /dev/null +++ b/storage-api/src/routes/internal.js @@ -0,0 +1,250 @@ +// Server-to-server endpoints for REN (never the browser). Every request must +// be HMAC-signed with the shared secret; REN signs the full path it calls +// (BASE_PATH included, which Apache keeps when proxying). Used by REN's pull +// reconciliation (dev, and as a safety net in prod) and for monitoring. +import fs from "node:fs/promises"; +import express from "express"; +import { config } from "../config.js"; +import { verifyRequest } from "../lib/storageSignature.js"; +import { incomingFile } from "../lib/paths.js"; +import { + findRecord, + readPending, + listPendingIds, + withLock, + closeRecord, +} from "../outbox/outbox.js"; +import { + applyDecision, + applyAck, + Permanent, + wakeWorker, +} from "../outbox/worker.js"; + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const router = express.Router(); + +router.use( + express.json({ + limit: "64kb", + verify: (req, res, buf) => { + req.rawBody = buf; + }, + }) +); +router.use((req, res, next) => { + const r = verifyRequest( + { + method: req.method, + path: req.originalUrl, + headers: req.headers, + rawBody: req.rawBody || "", + }, + config.hmacKeys + ); + if (!r.ok) { + console.warn("Rejected internal call:", r.reason, req.method, req.originalUrl); + return res.status(401).json({ error: "Invalid service signature." }); + } + next(); +}); + +const httpError = (status, message) => + Object.assign(new Error(message), { status }); + +const uploadId = (req) => { + const id = String(req.params.uploadId); + if (!UUID_RE.test(id)) throw httpError(404, "Unknown upload."); + return id; +}; + +const sendError = (res, err) => { + if (err instanceof Permanent) return res.status(409).json({ error: err.message }); + if (err.status) return res.status(err.status).json({ error: err.message }); + console.error("internal endpoint:", err.message); + res.status(500).json({ error: "Internal error." }); +}; + +const removeIncoming = async (id) => { + await fs.rm(incomingFile(id), { force: true }); + await fs.rm(`${incomingFile(id)}.json`, { force: true }); +}; + +// What the disk says about one upload. +const describe = async (id) => { + const found = await findRecord(id); + if (found) { + const r = found.rec; + return { + uploadId: id, + phase: r.state, + where: found.where, + size: r.size ?? r.expectedSize, + sha256: r.sha256 || null, + rawId: r.rawId || null, + storageKey: r.storageKey || null, + reason: r.reason || null, + lastError: r.lastError || null, + updatedAt: r.updatedAt, + }; + } + try { + const st = await fs.stat(incomingFile(id)); + let size = null; + try { + size = + JSON.parse(await fs.readFile(`${incomingFile(id)}.json`, "utf8")).size ?? + null; + } catch { + // no tus metadata + } + return { + uploadId: id, + phase: "incoming", + offset: st.size, + size, + lastActivityAt: st.mtime.toISOString(), + }; + } catch { + return { uploadId: id, phase: "none" }; + } +}; + +// GET /internal/uploads/:uploadId +router.get("/uploads/:uploadId", async (req, res) => { + try { + res.json(await describe(uploadId(req))); + } catch (err) { + sendError(res, err); + } +}); + +// POST /internal/uploads/:uploadId/decision +// {decision: "commit"|"discard", rawId?, storageKey?, reason?} +router.post("/uploads/:uploadId/decision", async (req, res) => { + try { + const id = uploadId(req); + const body = req.body || {}; + const state = await withLock(id, async () => { + const found = await findRecord(id); + if (!found) throw httpError(409, "Upload has not finished uploading."); + const rec = found.rec; + if (rec.state === "awaiting_decision" && found.where === "pending") { + await applyDecision(rec, body); + return rec.state; + } + // Repeat of a decision that already took effect → same answer, no change. + const committedAlready = [ + "committing", + "committed", + "discarding_committed", + "done", + ].includes(rec.state); + if (body.decision === "commit" && committedAlready && rec.rawId === body.rawId) { + return rec.state; + } + if ( + body.decision === "discard" && + ["discarding", "done"].includes(rec.state) && + !rec.rawId + ) { + return rec.state; + } + throw new Permanent(`cannot apply that decision in state ${rec.state}`); + }); + wakeWorker(); // committing / discarding continue right away + res.json({ ok: true, state }); + } catch (err) { + sendError(res, err); + } +}); + +// POST /internal/uploads/:uploadId/ack {action?: "keep"|"discard", reason?} +router.post("/uploads/:uploadId/ack", async (req, res) => { + try { + const id = uploadId(req); + const state = await withLock(id, async () => { + const found = await findRecord(id); + if (!found) throw httpError(404, "Unknown upload."); + if (found.where !== "pending") return found.rec.state; // finished → idempotent + await applyAck(found.rec, req.body || {}); + return found.rec.state; + }); + wakeWorker(); + res.json({ ok: true, state }); + } catch (err) { + sendError(res, err); + } +}); + +// DELETE /internal/uploads/:uploadId — leftovers of a cancelled/expired upload. +// Never removes a committed object (that's the cleanup job's decision). +router.delete("/uploads/:uploadId", async (req, res) => { + try { + const id = uploadId(req); + const result = await withLock(id, async () => { + const found = await findRecord(id); + if (found && found.where === "pending") { + if ( + ["committing", "committed", "discarding_committed"].includes( + found.rec.state + ) + ) { + throw new Permanent( + "upload is already committed; objects are removed by cleanup, not here" + ); + } + await removeIncoming(id); + found.rec.reason = "deleted by REN"; + found.rec.state = "done"; + await closeRecord(found.rec, "done"); + return "removed"; + } + if (found) return "nothing to remove"; // already done/failed + await removeIncoming(id); + return "removed"; + }); + res.json({ ok: true, result }); + } catch (err) { + sendError(res, err); + } +}); + +// GET /internal/status — for REN's monitoring (never public). +router.get("/status", async (req, res) => { + try { + const s = await fs.statfs(config.uploadRoot); + const byState = {}; + let oldest = null; + for (const id of await listPendingIds()) { + try { + const r = await readPending(id); + byState[r.state] = (byState[r.state] || 0) + 1; + if (!oldest || r.updatedAt < oldest.since) { + oldest = { uploadId: id, state: r.state, since: r.updatedAt }; + } + } catch { + // moved meanwhile + } + } + const count = async (sub) => + (await fs.readdir(`${config.outboxDir}/${sub}`)).filter((n) => + n.endsWith(".json") + ).length; + res.json({ + disk: { freeBytes: s.bavail * s.bsize, totalBytes: s.blocks * s.bsize }, + outbox: { + pending: byState, + done: await count("done"), + failed: await count("failed"), + }, + oldestPending: oldest, + pushes: config.mainApiUrl ? "on" : "off", + }); + } catch (err) { + sendError(res, err); + } +}); + +export default router; diff --git a/storage-api/src/server.js b/storage-api/src/server.js new file mode 100644 index 0000000..885e7a3 --- /dev/null +++ b/storage-api/src/server.js @@ -0,0 +1,53 @@ +// NeuroJSON storage API (Zodiac). Listens on 127.0.0.1; Apache adds TLS and +// proxies /neurojson-storage/* here: /health (public), /files (tus uploads, +// token-checked), /internal (REN only, HMAC-signed). +import fs from "node:fs/promises"; +import express from "express"; +import { config } from "./config.js"; +import { tusServer } from "./tus/tusServer.js"; +import { startWorker, stopWorker } from "./outbox/worker.js"; +import internalRoutes from "./routes/internal.js"; + +const app = express(); +app.disable("x-powered-by"); + +// Public health check: basic status only (no disk details — those are internal). +app.get(`${config.basePath}/health`, async (req, res) => { + try { + await fs.access(config.uploadRoot, fs.constants.W_OK); + res.json({ status: "ok" }); + } catch { + res.status(503).json({ status: "error" }); + } +}); + +// tus owns everything under /files (POST create, HEAD/PATCH/DELETE +// /files/, and the CORS preflight). No body parser: tus streams the body. +const tusHandler = (req, res) => tusServer.handle(req, res); +app.all(`${config.basePath}/files`, tusHandler); +app.all(`${config.basePath}/files/*splat`, tusHandler); + +// REN → Zodiac service calls (HMAC-signed): status, decisions, acks. +app.use(`${config.basePath}/internal`, internalRoutes); + +app.use((req, res) => res.status(404).json({ error: "Not found." })); + +const server = app.listen(config.port, config.host, () => { + console.log( + `storage-api listening on http://${config.host}:${config.port}${config.basePath}` + + ` | root ${config.uploadRoot}` + + ` | pushes ${ + config.mainApiUrl ? "ON → " + config.mainApiUrl : "OFF (pull mode)" + }` + ); + startWorker(); +}); + +// Let pm2 stop us cleanly (finish in-flight requests). Outbox records are on +// disk, so an interrupted step simply resumes on the next start. +for (const sig of ["SIGTERM", "SIGINT"]) { + process.on(sig, () => { + stopWorker(); + server.close(() => process.exit(0)); + }); +} diff --git a/storage-api/src/tus/tusServer.js b/storage-api/src/tus/tusServer.js new file mode 100644 index 0000000..a4e2565 --- /dev/null +++ b/storage-api/src/tus/tusServer.js @@ -0,0 +1,102 @@ +// tus resumable uploads into UPLOAD_ROOT/.incoming/. +// The upload id = the token's `uid` (assigned by REN), so URLs are +// /files/. When an upload finishes, an outbox record is +// written and the worker verifies it, asks REN, and moves it into place. +import fs from "node:fs/promises"; +import { Server } from "@tus/server"; +import { FileStore } from "@tus/file-store"; +import { config } from "../config.js"; +import { getClaims, tusError } from "../lib/tusAuth.js"; +import { hasRoomFor } from "../lib/diskSpace.js"; +import { incomingFile, rawObjectDir } from "../lib/paths.js"; +import { createRecord } from "../outbox/outbox.js"; +import { wakeWorker } from "../outbox/worker.js"; + +const exists = (p) => + fs.access(p).then( + () => true, + () => false + ); + +export const tusServer = new Server({ + path: `${config.basePath}/files`, + datastore: new FileStore({ + directory: config.incomingDir, + expirationPeriodInMilliseconds: 7 * 24 * 3600 * 1000, // idle uploads expire after 7 days + }), + respectForwardedHeaders: true, // Apache sets X-Forwarded-Proto/Host → https Location URLs + allowedOrigins: config.allowedOrigins, + allowedHeaders: ["Authorization"], + allowedCredentials: false, // Bearer token, no cookies + disableTerminationForFinishedUploads: true, + + // Runs FIRST on create: the token decides the upload id. + namingFunction: async (req) => getClaims(req).uid, + + // The upload can never be bigger than the size REN signed into the token. + maxSize: async (req, id) => + id ? getClaims(req).len : config.maxUploadBytes, + + // Every request: valid token, and the URL's upload id must be the token's. + onIncomingRequest: async (req, uploadId) => { + if (getClaims(req).uid !== uploadId) { + throw tusError(403, "This token is not for this upload."); + } + }, + + onUploadCreate: async (req, upload) => { + const c = getClaims(req); + if (upload.size === undefined) { + throw tusError(400, "The file size must be known when the upload starts."); + } + if (upload.size !== c.len) { + throw tusError( + 400, + "Upload-Length must equal the size declared to NeuroJSON." + ); + } + // Never restart an upload that already has data (that would wipe + // progress), and never re-create one that already finished. + if ( + (await exists(incomingFile(c.uid))) || + (await exists(rawObjectDir(c.iid, c.uid))) + ) { + throw tusError(409, "This upload already exists. Resume it instead."); + } + if (!(await hasRoomFor(upload.size, c.uid))) { + throw tusError(507, "Not enough storage space for this file right now."); + } + // Server-side facts for the finish step; these win over client metadata. + return { + metadata: { + ...upload.metadata, + nj_internal_id: c.iid, + nj_submission_id: c.sid, + nj_upload_id: c.uid, + }, + }; + }, + + onUploadFinish: async (req, upload) => { + // Values set by the server in onUploadCreate (never client-supplied). + const m = upload.metadata || {}; + await createRecord({ + uploadId: upload.id, + internalId: m.nj_internal_id, + submissionId: m.nj_submission_id, + expectedSize: upload.size, + state: "verifying", + }); + console.log(`upload complete: ${upload.id} (${upload.size} bytes)`); + wakeWorker(); + return {}; // reply to the browser right away; checks run in the background + }, + + // Our own errors ({status_code, body}) pass through; anything unexpected + // becomes a generic 500 so file paths / stack details never reach the client. + onResponseError: async (req, err) => { + if (err && typeof err.status_code === "number") return undefined; + console.error("tus error:", err?.message || err); + return { status_code: 500, body: "Internal server error.\n" }; + }, +}); diff --git a/yarn.lock b/yarn.lock index b91968c..b993abe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4293,7 +4293,7 @@ bser@2.1.1: dependencies: node-int64 "^0.4.0" -buffer-from@^1.0.0: +buffer-from@^1.0.0, buffer-from@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== @@ -4664,6 +4664,14 @@ colorette@^2.0.10: resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.20.tgz#9eb793e6833067f7235902fcd3b09917a000a95a" integrity sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w== +combine-errors@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/combine-errors/-/combine-errors-3.0.3.tgz#f4df6740083e5703a3181110c2b10551f003da86" + integrity sha512-C8ikRNRMygCwaTx+Ek3Yr+OuZzgZjduCOfSQBjbM8V3MfgcjSTeto/GXP6PAwKvJz/v15b7GHZvx5rOlczFw/Q== + dependencies: + custom-error-instance "2.1.1" + lodash.uniqby "4.5.0" + combined-stream@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" @@ -5230,6 +5238,11 @@ csstype@^3.0.2, csstype@^3.1.3: resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81" integrity sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw== +custom-error-instance@2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/custom-error-instance/-/custom-error-instance-2.1.1.tgz#3cf6391487a6629a6247eb0ca0ce00081b7e361a" + integrity sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg== + cwise-compiler@^1.0.0, cwise-compiler@^1.1.1: version "1.1.3" resolved "https://registry.yarnpkg.com/cwise-compiler/-/cwise-compiler-1.1.3.tgz#f4d667410e850d3a313a7d2db7b1e505bb034cc5" @@ -8506,6 +8519,11 @@ jquery@^3.7.1: resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.7.1.tgz#083ef98927c9a6a74d05a6af02806566d16274de" integrity sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg== +js-base64@^3.7.2: + version "3.9.4" + resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-3.9.4.tgz#bc4d811fd52105848df55bc80ca401d963020e6d" + integrity sha512-PtOMXpEGuP0RRiRXsjzHzl44dMHxSu2CPvAhinupR1tBa88me+1DPqsobl7eupn5UukjLkln1ZnAOAOXOrYG0Q== + "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -8849,6 +8867,43 @@ lodash-es@4, lodash-es@^4.17.21: resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee" integrity sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw== +lodash._baseiteratee@~4.7.0: + version "4.7.0" + resolved "https://registry.yarnpkg.com/lodash._baseiteratee/-/lodash._baseiteratee-4.7.0.tgz#34a9b5543572727c3db2e78edae3c0e9e66bd102" + integrity sha512-nqB9M+wITz0BX/Q2xg6fQ8mLkyfF7MU7eE+MNBNjTHFKeKaZAPEzEg+E8LWxKWf1DQVflNEn9N49yAuqKh2mWQ== + dependencies: + lodash._stringtopath "~4.8.0" + +lodash._basetostring@~4.12.0: + version "4.12.0" + resolved "https://registry.yarnpkg.com/lodash._basetostring/-/lodash._basetostring-4.12.0.tgz#9327c9dc5158866b7fa4b9d42f4638e5766dd9df" + integrity sha512-SwcRIbyxnN6CFEEK4K1y+zuApvWdpQdBHM/swxP962s8HIxPO3alBH5t3m/dl+f4CMUug6sJb7Pww8d13/9WSw== + +lodash._baseuniq@~4.6.0: + version "4.6.0" + resolved "https://registry.yarnpkg.com/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz#0ebb44e456814af7905c6212fa2c9b2d51b841e8" + integrity sha512-Ja1YevpHZctlI5beLA7oc5KNDhGcPixFhcqSiORHNsp/1QTv7amAXzw+gu4YOvErqVlMVyIJGgtzeepCnnur0A== + dependencies: + lodash._createset "~4.0.0" + lodash._root "~3.0.0" + +lodash._createset@~4.0.0: + version "4.0.3" + resolved "https://registry.yarnpkg.com/lodash._createset/-/lodash._createset-4.0.3.tgz#0f4659fbb09d75194fa9e2b88a6644d363c9fe26" + integrity sha512-GTkC6YMprrJZCYU3zcqZj+jkXkrXzq3IPBcF/fIPpNEAB4hZEtXU8zp/RwKOvZl43NUmwDbyRk3+ZTbeRdEBXA== + +lodash._root@~3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/lodash._root/-/lodash._root-3.0.1.tgz#fba1c4524c19ee9a5f8136b4609f017cf4ded692" + integrity sha512-O0pWuFSK6x4EXhM1dhZ8gchNtG7JMqBtrHdoUFUWXD7dJnNSUze1GuyQr5sOs0aCvgGeI3o/OJW8f4ca7FDxmQ== + +lodash._stringtopath@~4.8.0: + version "4.8.0" + resolved "https://registry.yarnpkg.com/lodash._stringtopath/-/lodash._stringtopath-4.8.0.tgz#941bcf0e64266e5fc1d66fed0a6959544c576824" + integrity sha512-SXL66C731p0xPDC5LZg4wI5H+dJo/EO4KTqOMwLYCH3+FmmfAKJEZCm6ohGpI+T1xwsDsJCfL4OnhorllvlTPQ== + dependencies: + lodash._basetostring "~4.12.0" + lodash.curry@^4.0.1: version "4.1.1" resolved "https://registry.yarnpkg.com/lodash.curry/-/lodash.curry-4.1.1.tgz#248e36072ede906501d75966200a86dab8b23170" @@ -8879,11 +8934,24 @@ lodash.sortby@^4.7.0: resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" integrity sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA== +lodash.throttle@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/lodash.throttle/-/lodash.throttle-4.1.1.tgz#c23e91b710242ac70c37f1e1cda9274cc39bf2f4" + integrity sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ== + lodash.uniq@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== +lodash.uniqby@4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.uniqby/-/lodash.uniqby-4.5.0.tgz#a3a17bbf62eeb6240f491846e97c1c4e2a5e1e21" + integrity sha512-IRt7cfTtHy6f1aRVA5n7kT8rgN3N1nH6MOWLcHfpWG2SH19E3JksLK38MktLxZDhlAjCP9jpIXkOnRXlu6oByQ== + dependencies: + lodash._baseiteratee "~4.7.0" + lodash._baseuniq "~4.6.0" + lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.4, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" @@ -10510,6 +10578,15 @@ prop-types@^15.6.2, prop-types@^15.8.1: object-assign "^4.1.1" react-is "^16.13.1" +proper-lockfile@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/proper-lockfile/-/proper-lockfile-4.1.2.tgz#c8b9de2af6b2f1601067f98e01ac66baa223141f" + integrity sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA== + dependencies: + graceful-fs "^4.2.4" + retry "^0.12.0" + signal-exit "^3.0.2" + property-information@^7.0.0: version "7.1.0" resolved "https://registry.yarnpkg.com/property-information/-/property-information-7.1.0.tgz#b622e8646e02b580205415586b40804d3e8bfd5d" @@ -11118,6 +11195,11 @@ resolve@^2.0.0-next.5: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" +retry@^0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b" + integrity sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow== + retry@^0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658" @@ -12415,6 +12497,19 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +tus-js-client@^4.3.1: + version "4.3.1" + resolved "https://registry.yarnpkg.com/tus-js-client/-/tus-js-client-4.3.1.tgz#2f9a47fba006206fb0d08c649fa01254944d5d87" + integrity sha512-ZLeYmjrkaU1fUsKbIi8JML52uAocjEZtBx4DKjRrqzrZa0O4MYwT6db+oqePlspV+FxXJAyFBc/L5gwUi2OFsg== + dependencies: + buffer-from "^1.1.2" + combine-errors "^3.0.3" + is-stream "^2.0.0" + js-base64 "^3.7.2" + lodash.throttle "^4.1.1" + proper-lockfile "^4.1.2" + url-parse "^1.5.7" + type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" @@ -12669,7 +12764,7 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" -url-parse@^1.5.3: +url-parse@^1.5.3, url-parse@^1.5.7: version "1.5.10" resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==