feat(uploads): dataset upload with drafts, review workflow, and resumable raw ZIP uploads; refs #147

.gitignore

@@ -16,4 +16,8 @@ backend/.env.local
 backend/.env.*.local
 backend/*.sqlite
 backend/*.db
-!backend/package-lock.json
+!backend/package-lock.json
+
+#storage-api
+storage-api/storage-dev/
+!storage-api/package-lock.json

backend/migrations/20260923192128-create-dataset-registry.js

@@ -0,0 +1,64 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // Permanent record for one logical dataset. `internal_id` is the stable
+  // identity for the dataset's whole lifetime and is also the sandbox CouchDB
+  // `_id`. `dataset_id` (final public id) is assigned only at promotion.
+  async up(queryInterface, Sequelize) {
+    // Sequence backing NeuroJSON-generated public ids (njds######). Consumed
+    // only at promotion when the user did not request a valid custom id.
+    await queryInterface.sequelize.query(
+      "CREATE SEQUENCE IF NOT EXISTS neurojson_dataset_seq START 1;"
+    );
+
+    await queryInterface.createTable("dataset_registry", {
+      internal_id: {
+        type: Sequelize.UUID,
+        primaryKey: true,
+        allowNull: false, // app-generated (crypto.randomUUID); also the sandbox _id
+      },
+      dataset_id: {
+        type: Sequelize.STRING(255),
+        allowNull: true,
+        unique: true, // final public id, set at promotion (njds###### or custom)
+      },
+      requested_dataset_id: {
+        type: Sequelize.STRING(63),
+        allowNull: true, // user's preferred public id — a preference until promotion
+      },
+      dataset_name: {
+        type: Sequelize.STRING(255),
+        allowNull: true, // canonical logical-dataset name
+      },
+      owner_user_id: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        references: { model: "users", key: "id" },
+        onDelete: "RESTRICT", // don't orphan a dataset's owner
+        onUpdate: "CASCADE",
+      },
+      created_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+      updated_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+    });
+
+    await queryInterface.addIndex("dataset_registry", ["owner_user_id"], {
+      name: "idx_registry_owner",
+    });
+  },
+
+  async down(queryInterface) {
+    await queryInterface.dropTable("dataset_registry");
+    await queryInterface.sequelize.query(
+      "DROP SEQUENCE IF EXISTS neurojson_dataset_seq;"
+    );
+  },
+};

backend/migrations/20260923192138-create-submissions.js

@@ -0,0 +1,75 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // One review workflow (cycle) for a logical dataset. Holds only the current
+  // status + review/promotion metadata; the reviewer<->submitter discussion
+  // lives in submission_comments. A dataset can have many cycles over time,
+  // but only ONE open one (see the partial unique index below).
+  async up(queryInterface, Sequelize) {
+    await queryInterface.createTable("submissions", {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false,
+      },
+      submission_id: {
+        type: Sequelize.UUID,
+        allowNull: false,
+        unique: true, // one review cycle; referenced by submission_comments
+      },
+      dataset_registry_id: {
+        type: Sequelize.UUID,
+        allowNull: false,
+        references: { model: "dataset_registry", key: "internal_id" },
+        onDelete: "CASCADE", // removing the logical dataset removes its cycles
+        onUpdate: "CASCADE",
+      },
+      status: {
+        type: Sequelize.STRING(50),
+        allowNull: false,
+        defaultValue: "pending", // pending|changes_requested|approved|rejected|promoted
+      },
+      reviewed_by: {
+        type: Sequelize.INTEGER,
+        allowNull: true,
+        references: { model: "users", key: "id" },
+        onDelete: "SET NULL", // keep the cycle if a reviewer is removed
+        onUpdate: "CASCADE",
+      },
+      reviewed_at: { type: Sequelize.DATE, allowNull: true },
+      promoted_db: { type: Sequelize.STRING(255), allowNull: true },
+      promoted_at: { type: Sequelize.DATE, allowNull: true },
+      created_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+      updated_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+    });
+
+    await queryInterface.addIndex("submissions", ["dataset_registry_id"], {
+      name: "idx_submissions_registry",
+    });
+    await queryInterface.addIndex("submissions", ["status"], {
+      name: "idx_submissions_status",
+    });
+
+    // At most one OPEN workflow per dataset. "Open" = still active:
+    // pending / changes_requested / approved (approved is awaiting promotion,
+    // so it must also block a new cycle). rejected/promoted are terminal, so a
+    // confirmed resubmit can insert a fresh pending row.
+    await queryInterface.sequelize.query(
+      "CREATE UNIQUE INDEX uq_one_open_per_dataset ON submissions (dataset_registry_id) WHERE status IN ('pending','changes_requested','approved');"
+    );
+  },
+
+  async down(queryInterface) {
+    await queryInterface.dropTable("submissions");
+  },
+};

backend/migrations/20260923192147-create-submission-comments.js

@@ -0,0 +1,50 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // Chronological reviewer <-> submitter discussion for a submission cycle,
+  // one row per comment. This is the single source of review discussion
+  // (there is no review_note column). Author role is derived at read time
+  // (user_id == dataset_registry.owner_user_id => submitter, else reviewer).
+  async up(queryInterface, Sequelize) {
+    await queryInterface.createTable("submission_comments", {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false,
+      },
+      submission_id: {
+        type: Sequelize.UUID,
+        allowNull: false,
+        references: { model: "submissions", key: "submission_id" },
+        onDelete: "CASCADE", // comments die with their cycle
+        onUpdate: "CASCADE",
+      },
+      user_id: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        references: { model: "users", key: "id" },
+        onDelete: "RESTRICT", // a comment must keep a real author
+        onUpdate: "CASCADE",
+      },
+      message: { type: Sequelize.TEXT, allowNull: false },
+      created_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+    });
+
+    // Serves the read query: WHERE submission_id = ? ORDER BY created_at ASC.
+    await queryInterface.addIndex(
+      "submission_comments",
+      ["submission_id", "created_at"],
+      { name: "idx_comments_submission_created" }
+    );
+  },
+
+  async down(queryInterface) {
+    await queryInterface.dropTable("submission_comments");
+  },
+};

backend/migrations/20260923214452-add-unique-requested-dataset-id.js

@@ -0,0 +1,19 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // At most one dataset may reserve a given preferred public id (first-come).
+  // Partial so the many NULL (no preference) rows don't collide. Backs up the
+  // controller's assertRequestedIdFree check against races.
+  async up(queryInterface) {
+    await queryInterface.sequelize.query(
+      "CREATE UNIQUE INDEX uq_requested_dataset_id ON dataset_registry (requested_dataset_id) WHERE requested_dataset_id IS NOT NULL;"
+    );
+  },
+
+  async down(queryInterface) {
+    await queryInterface.sequelize.query(
+      "DROP INDEX IF EXISTS uq_requested_dataset_id;"
+    );
+  }
+};

backend/migrations/20260925162336-add-requested-db-to-dataset-registry.js

@@ -0,0 +1,19 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // The public database a dataset should be published to at promotion.
+  // A preference/reservation; defaults to "public" when the user picks nothing.
+  // Actual db is resolved/created at promotion (a new name is review-gated).
+  async up(queryInterface, Sequelize) {
+    await queryInterface.addColumn("dataset_registry", "requested_db", {
+      type: Sequelize.STRING(63),
+      allowNull: false,
+      defaultValue: "public",
+    });
+  },
+
+  async down(queryInterface) {
+    await queryInterface.removeColumn("dataset_registry", "requested_db");
+  }
+};

backend/migrations/20260928210352-add-draft-workflow-to-submissions.js

@@ -0,0 +1,75 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // Draft workflow: review status starts at 'draft'; JSON upload state is
+  // tracked separately from review state; raw ZIP is optional per submission.
+  async up(queryInterface, Sequelize) {
+    // New cycles start as draft (not pending).
+    await queryInterface.changeColumn("submissions", "status", {
+      type: Sequelize.STRING(50),
+      allowNull: false,
+      defaultValue: "draft", // draft|pending|changes_requested|approved|rejected|promoted
+    });
+
+    // JSON component state (separate from review status).
+    await queryInterface.addColumn("submissions", "json_status", {
+      type: Sequelize.STRING(20),
+      allowNull: false,
+      defaultValue: "uploaded", // uploaded | failed
+    });
+    await queryInterface.addColumn("submissions", "json_uploaded_at", {
+      type: Sequelize.DATE,
+      allowNull: true,
+    });
+    await queryInterface.addColumn("submissions", "json_error", {
+      type: Sequelize.TEXT,
+      allowNull: true,
+    });
+
+    // Last time the user sent it to review (draft/changes_requested → pending).
+    await queryInterface.addColumn("submissions", "submitted_at", {
+      type: Sequelize.DATE,
+      allowNull: true,
+    });
+
+    // Raw ZIP is optional; required for submit only when this is true.
+    await queryInterface.addColumn("submissions", "raw_zip_expected", {
+      type: Sequelize.BOOLEAN,
+      allowNull: false,
+      defaultValue: false,
+    });
+
+    // Existing rows already had their JSON uploaded.
+    await queryInterface.sequelize.query(
+      "UPDATE submissions SET json_uploaded_at = updated_at WHERE json_uploaded_at IS NULL;"
+    );
+
+    // One active cycle per dataset — now including draft.
+    await queryInterface.sequelize.query(
+      "DROP INDEX IF EXISTS uq_one_open_per_dataset;"
+    );
+    await queryInterface.sequelize.query(
+      "CREATE UNIQUE INDEX uq_one_open_per_dataset ON submissions (dataset_registry_id) WHERE status IN ('draft','pending','changes_requested','approved');"
+    );
+  },
+
+  async down(queryInterface, Sequelize) {
+    await queryInterface.sequelize.query(
+      "DROP INDEX IF EXISTS uq_one_open_per_dataset;"
+    );
+    await queryInterface.sequelize.query(
+      "CREATE UNIQUE INDEX uq_one_open_per_dataset ON submissions (dataset_registry_id) WHERE status IN ('pending','changes_requested','approved');"
+    );
+    await queryInterface.removeColumn("submissions", "raw_zip_expected");
+    await queryInterface.removeColumn("submissions", "submitted_at");
+    await queryInterface.removeColumn("submissions", "json_error");
+    await queryInterface.removeColumn("submissions", "json_uploaded_at");
+    await queryInterface.removeColumn("submissions", "json_status");
+    await queryInterface.changeColumn("submissions", "status", {
+      type: Sequelize.STRING(50),
+      allowNull: false,
+      defaultValue: "pending",
+    });
+  },
+};