adapter: normalize Foundry Sets & pseudo-doc collections to JSON (Phase C)

scripts/adapters/generic-adapter.mjs

@@ -25,6 +25,12 @@
  *   - type:              "json"/"string" → serialized JSON string; else a raw array
  *   Collection fields are READ-ONLY today (pull only); write-back is a future tier,
  *   so they are never included in the Foundry update path.
+ *
+ * Every extracted value (scalar or projected) is run through
+ * `normalizeFoundryValue` so live Foundry structures that `JSON.stringify`
+ * cannot serialize — Sets (keywords, skills, characteristics, actor.statuses)
+ * and Collections of pseudo-documents (an ability's `system.power.effects` tier
+ * ladder) — become plain arrays/objects instead of `{}`.
  */
 
 /**
@@ -145,6 +151,68 @@ export function getNestedValue(obj, path) {
   return current;
 }
 
+/**
+ * Normalize a value read off a live Foundry document into a JSON-safe plain
+ * value. Foundry models many fields as data structures that `JSON.stringify`
+ * cannot serialize meaningfully:
+ *   - a **Set** (keywords, skills, power-roll characteristics, actor.statuses)
+ *     serializes to `{}` — must become an array;
+ *   - a **Collection / ModelCollection** (a Map subclass — e.g. a CollectionField
+ *     of pseudo-documents like an ability's `system.power.effects` tier ladder)
+ *     also serializes to `{}` — must become an array of its members;
+ *   - a **DataModel / pseudo-document** member exposes its data via `toObject()`,
+ *     not own-enumerable properties (the schema fields are getters).
+ *
+ * This walks such structures recursively and returns arrays / plain objects /
+ * primitives only. It is defensive (never throws) and depth-guarded against
+ * cycles. Primitives and already-plain values pass straight through, so it is
+ * safe to apply to every extracted field.
+ *
+ * @param {*} value
+ * @param {number} [depth]
+ * @returns {*} a JSON-safe value (primitive | array | plain object | null)
+ */
+export function normalizeFoundryValue(value, depth) {
+  depth = depth || 0;
+  if (value == null) return value;
+  if (typeof value !== 'object') return value; // primitives pass through
+  if (depth > 8) return null;                   // cycle / runaway guard
+
+  // Native Set OR Foundry Set → array of normalized members.
+  if (value instanceof Set) {
+    return Array.from(value, (v) => normalizeFoundryValue(v, depth + 1));
+  }
+  // Native array → map members.
+  if (Array.isArray(value)) {
+    return value.map((v) => normalizeFoundryValue(v, depth + 1));
+  }
+  // Foundry Collection (a Map subclass) exposes its members as `.contents`.
+  // This covers CollectionField values (pseudo-document tier ladders) and the
+  // actor's items/effects collections when a path points straight at one.
+  if (Array.isArray(value.contents)) {
+    return value.contents.map((v) => normalizeFoundryValue(v, depth + 1));
+  }
+  // Plain Map (no `.contents`) → array of its values.
+  if (value instanceof Map) {
+    return Array.from(value.values(), (v) => normalizeFoundryValue(v, depth + 1));
+  }
+  // DataModel / pseudo-document → its source object (schema fields are getters,
+  // not own-enumerable props, so a key-copy would miss them). `toObject(false)`
+  // yields the stored source (Sets already serialized to arrays); re-normalize
+  // it to catch any nested live structures.
+  if (typeof value.toObject === 'function') {
+    try {
+      return normalizeFoundryValue(value.toObject(false), depth + 1);
+    } catch (e) { /* fall through to a shallow copy */ }
+  }
+  // Plain-ish object → normalize own enumerable props (catches nested Sets).
+  const out = {};
+  for (const k of Object.keys(value)) {
+    out[k] = normalizeFoundryValue(value[k], depth + 1);
+  }
+  return out;
+}
+
 /**
  * Extract a collection-mapped field (e.g. abilities/inventory from actor.items[]).
  * Reads field.foundry_collection off the actor, optionally filters by
@@ -187,7 +255,7 @@ export function extractCollectionField(actor, field) {
       if (!first) return '';
       if (proj) {
         const firstPath = Object.values(proj)[0];
-        return getNestedValue(first, firstPath) ?? '';
+        return normalizeFoundryValue(getNestedValue(first, firstPath)) ?? '';
       }
       return first.name ?? '';
     }
@@ -196,7 +264,7 @@ export function extractCollectionField(actor, field) {
       if (!proj) return { id: it.id ?? null, name: it.name ?? null, type: it.type ?? null };
       const out = {};
       for (const [outKey, path] of Object.entries(proj)) {
-        out[outKey] = getNestedValue(it, path) ?? null;
+        out[outKey] = normalizeFoundryValue(getNestedValue(it, path)) ?? null;
       }
       return out;
     });
@@ -223,7 +291,14 @@ export function buildChronicleFields(actor, mappedFields) {
     if (field.foundry_collection) {
       result[field.key] = extractCollectionField(actor, field);
     } else {
-      const value = getNestedValue(actor, field.foundry_path);
+      let value = normalizeFoundryValue(getNestedValue(actor, field.foundry_path));
+      // A Set/array/object scalar (e.g. system.skills.value, actor.statuses)
+      // declared on a string/json field is serialized to a JSON string so it
+      // arrives as parseable JSON, mirroring how collection fields are stored.
+      if (value != null && typeof value === 'object'
+          && (field.type === 'string' || field.type === 'json')) {
+        value = JSON.stringify(value);
+      }
       result[field.key] = value ?? null;
     }
   }

scripts/sync-dashboard.mjs

@@ -66,6 +66,7 @@ export class SyncDashboard extends HandlebarsApplicationMixin(ApplicationV2) {
       'open-map-journal': SyncDashboard.#onOpenMapJournalAction,
       'resync-all-maps': SyncDashboard.#onResyncAllMapsAction,
       'resync-all-journals': SyncDashboard.#onResyncAllJournalsAction,
+      'resync-everything': SyncDashboard.#onResyncEverythingAction,
       'open-maps-folder': SyncDashboard.#onOpenMapsFolderAction,
       'dismiss-map-errors': SyncDashboard.#onDismissMapErrorsAction,
       'pull-date': SyncDashboard.#onPullDateAction,
@@ -1469,6 +1470,11 @@ export class SyncDashboard extends HandlebarsApplicationMixin(ApplicationV2) {
     await this._onResyncAllJournals();
   }
 
+  /** Re-sync EVERYTHING: journals + every linked character + maps. */
+  static async #onResyncEverythingAction() {
+    await this._onResyncEverything();
+  }
+
   /**
    * Reveal the "Chronicle Maps" folder in Foundry's journal sidebar.
    * Activates the journal tab and expands the folder.
@@ -2172,6 +2178,71 @@ export class SyncDashboard extends HandlebarsApplicationMixin(ApplicationV2) {
     this.render({ force: true });
   }
 
+  /**
+   * Re-sync everything that's already connected to Chronicle in one action:
+   * all journals, every LINKED character (re-pushes current field data — Foundry
+   * is source of truth for characters), and all maps. This is the Overview's
+   * "Sync Everything Now"; the previous button only did journals, which left
+   * actor field data (and inventory/notes) stale after a manifest/path change.
+   *
+   * Unlinked actors are intentionally NOT auto-created here — that's a heavier,
+   * entity-creating operation kept on the Characters tab's "Push All Actors".
+   * Each module's resyncAll / repushActor is the same proven path the per-tab
+   * buttons use; failures are isolated per item so one bad actor can't abort the
+   * sweep.
+   */
+  async _onResyncEverything() {
+    let confirmed;
+    try {
+      confirmed = await confirmDialog({
+        title: 'Sync Everything Now',
+        content: '<p>Re-sync <strong>all</strong> journals, linked characters, and maps with Chronicle?</p>'
+          + '<p class="hint">Foundry is the source of truth for characters — their current data is pushed up.</p>',
+      });
+    } catch {
+      return; // User dismissed the dialog.
+    }
+    if (!confirmed) return;
+
+    ui.notifications.info('Chronicle: Re-syncing everything…');
+    const mods = this._syncManager?._modules || [];
+    const find = (name) => mods.find((m) => m.constructor?.name === name);
+    const done = [];
+
+    // 1. Journals (verbose=false to avoid a flood of per-entry toasts mid-sweep).
+    const journalSync = find('JournalSync');
+    if (journalSync?.resyncAll) {
+      try { await journalSync.resyncAll({ verbose: false }); done.push('journals'); }
+      catch (err) { console.error('Chronicle: journal resync failed', err); }
+    }
+
+    // 2. Characters — re-push every LINKED actor so stale fields refresh.
+    const actorSync = find('ActorSync');
+    if (actorSync?.repushActor) {
+      const chars = actorSync.getSyncedActors?.() ?? [];
+      let pushed = 0;
+      for (const c of chars) {
+        if (!c.synced) continue; // unlinked → use "Push All Actors" (creates entities)
+        try { if (await actorSync.repushActor(c.id)) pushed++; }
+        catch (err) { console.error(`Chronicle: re-push failed for "${c.name}"`, err); }
+      }
+      if (pushed) done.push(`${pushed} character(s)`);
+    }
+
+    // 3. Maps.
+    const mapSync = find('MapSync');
+    if (mapSync?.resyncAll) {
+      try { await mapSync.resyncAll({ verbose: false }); done.push('maps'); }
+      catch (err) { console.error('Chronicle: map resync failed', err); }
+    }
+
+    this._cache.maps = null;
+    this._cache.entities = null;
+    this._logActivity('push', `Re-synced everything (${done.join(', ') || 'nothing'})`);
+    this.render({ force: true });
+    ui.notifications.info(`Chronicle: Re-sync complete — ${done.join(', ') || 'nothing to sync'}.`);
+  }
+
   // ---------------------------------------------------------------------------
   // Config actions
   // ---------------------------------------------------------------------------

templates/sync-dashboard.hbs

@@ -138,8 +138,8 @@
         {{!-- Quick actions --}}
         <h3 class="section-heading">Quick Actions</h3>
         <div class="overview-actions">
-          <button type="button" class="dashboard-btn btn-sm" data-action="resync-all-journals">
-            <i class="fa-solid fa-rotate"></i> Sync Journals Now
+          <button type="button" class="dashboard-btn btn-sm" data-action="resync-everything">
+            <i class="fa-solid fa-rotate"></i> Sync Everything Now
           </button>
           <button type="button" class="dashboard-btn btn-sm" data-jump-tab="status">
             <i class="fa-solid fa-stethoscope"></i> Diagnostics

tools/test-generic-adapter.mjs

@@ -12,9 +12,25 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 
-const { getNestedValue, extractCollectionField, buildChronicleFields } =
+const { getNestedValue, extractCollectionField, buildChronicleFields, normalizeFoundryValue } =
   await import('../scripts/adapters/generic-adapter.mjs');
 
+// ── Foundry-ish data structures for the normalizer tests ──────────────────
+// A Foundry Collection is a Map subclass exposing members via `.contents`.
+class FakeCollection extends Map {
+  constructor(members) {
+    super(members.map((m, i) => [m._id ?? String(i), m]));
+  }
+  get contents() { return Array.from(this.values()); }
+}
+// A pseudo-document / DataModel exposes data via toObject(), not own props —
+// the schema fields here are GETTERS so a naive key-copy would miss them.
+class FakeModel {
+  constructor(source) { this._source = source; }
+  get tier1() { return this._source.tier1; }
+  toObject() { return JSON.parse(JSON.stringify(this._source)); }
+}
+
 // A Foundry-ish actor with system data + an items collection (Map-like w/ .contents).
 function makeActor() {
   return {
@@ -134,3 +150,69 @@ test('buildChronicleFields sets null for a missing scalar path', () => {
   const out = buildChronicleFields(makeActor(), [{ key: 'speed', foundry_path: 'system.movement.speed', type: 'number' }]);
   assert.equal(out.speed, null);
 });
+
+// ── normalizeFoundryValue ─────────────────────────────────────────────────
+
+test('normalizeFoundryValue passes primitives and plain values through', () => {
+  assert.equal(normalizeFoundryValue(7), 7);
+  assert.equal(normalizeFoundryValue('hi'), 'hi');
+  assert.equal(normalizeFoundryValue(null), null);
+  assert.equal(normalizeFoundryValue(undefined), undefined);
+  assert.deepEqual(normalizeFoundryValue([1, 2, 3]), [1, 2, 3]);
+  assert.deepEqual(normalizeFoundryValue({ a: 1, b: 'x' }), { a: 1, b: 'x' });
+});
+
+test('normalizeFoundryValue turns a Set into an array (keywords/skills/statuses)', () => {
+  assert.deepEqual(normalizeFoundryValue(new Set(['magic', 'ranged'])), ['magic', 'ranged']);
+  assert.deepEqual(normalizeFoundryValue(new Set()), []);
+});
+
+test('normalizeFoundryValue normalizes nested Sets inside a plain object', () => {
+  const v = normalizeFoundryValue({ type: 'damage', types: new Set(['fire']), n: 2 });
+  assert.deepEqual(v, { type: 'damage', types: ['fire'], n: 2 });
+});
+
+test('normalizeFoundryValue unrolls a Foundry Collection via .contents + toObject', () => {
+  const effects = new FakeCollection([
+    new FakeModel({ _id: 'a', type: 'damage', tier1: { value: '2 + @chr', types: ['fire'] } }),
+    new FakeModel({ _id: 'b', type: 'damage', tier1: { value: '3', types: [] } }),
+  ]);
+  const out = normalizeFoundryValue(effects);
+  assert.ok(Array.isArray(out));
+  assert.equal(out.length, 2);
+  assert.deepEqual(out[0], { _id: 'a', type: 'damage', tier1: { value: '2 + @chr', types: ['fire'] } });
+  assert.equal(out[1].tier1.value, '3');
+});
+
+test('extractCollectionField projects a Set sub-field to a JSON array (ability keywords)', () => {
+  const actor = {
+    items: { contents: [
+      { id: 'i1', name: 'Ray of Wrath', type: 'ability',
+        system: { type: 'main', category: 'signature', keywords: new Set(['magic', 'ranged']) } },
+    ] },
+  };
+  const json = extractCollectionField(actor, {
+    key: 'abilities_json', type: 'string',
+    foundry_collection: 'items', foundry_item_type: 'ability',
+    foundry_item_fields: { name: 'name', type: 'system.type', keywords: 'system.keywords' },
+  });
+  const abilities = JSON.parse(json);
+  assert.deepEqual(abilities[0], { name: 'Ray of Wrath', type: 'main', keywords: ['magic', 'ranged'] });
+});
+
+test('buildChronicleFields serializes a Set scalar on a string field to JSON (skills)', () => {
+  const actor = { system: { skills: { value: new Set(['alchemy', 'timescape']) } } };
+  const out = buildChronicleFields(actor, [
+    { key: 'skills_json', foundry_path: 'system.skills.value', type: 'string' },
+  ]);
+  assert.equal(typeof out.skills_json, 'string');
+  assert.deepEqual(JSON.parse(out.skills_json), ['alchemy', 'timescape']);
+});
+
+test('buildChronicleFields serializes actor.statuses (Set) for conditions', () => {
+  const actor = { system: {}, statuses: new Set(['slowed', 'weakened']) };
+  const out = buildChronicleFields(actor, [
+    { key: 'conditions_json', foundry_path: 'statuses', type: 'string' },
+  ]);
+  assert.deepEqual(JSON.parse(out.conditions_json), ['slowed', 'weakened']);
+});