Merge pull request #72 from botandrose/two-pass

Add `twoPass` option for additional state retention

.github/workflows/ci.yml

@@ -33,3 +33,37 @@ jobs:
       - name: Run tests
         run: npm run test-move-before
 
+  test-force-two-pass:
+    runs-on: ubuntu-latest
+    env:
+      DEFAULT_TWO_PASS: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          cache: 'npm'
+      - name: Install dependencies
+        run: npm install
+      - name: Install browsers
+        run: npx playwright install --with-deps
+      - name: Run tests
+        run: npm run ci
+
+  test-force-two-pass-move-before:
+    runs-on: ubuntu-latest
+    env:
+      DEFAULT_TWO_PASS: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          cache: 'npm'
+      - name: Install dependencies
+        run: npm install
+      - name: Run tests
+        run: npm run test-move-before
+

README.md

@@ -96,6 +96,7 @@ Idiomorph supports the following options:
 | `ignoreActiveValue` | If set to `true`, idiomorph will not update the active element's value                                      | `Idiomorph.morph(..., {ignoreActiveValue:true})`                         |
 | `head`              | Allows you to control how the `head` tag is merged.  See the [head](#the-head-tag) section for more details | `Idiomorph.morph(..., {head:{style:merge}})`                             |
 | `callbacks`         | Allows you to insert callbacks when events occur in the morph life cycle, see the callback table below      | `Idiomorph.morph(..., {callbacks:{beforeNodeAdded:function(node){...}})` |
+| `twoPass`           | If set to `true`, idiomorph does a second pass to maintain more state. See [Two Pass Mode](#two-pass-mode)  | `Idiomorph.morph(..., {twoPass:true})`                                   |
 
 #### Callbacks
 
@@ -110,7 +111,8 @@ of the algorithm.
 | afterNodeMorphed(oldNode, newNode)                           | Called after a node is morphed in the DOM                                                | none                                               |
 | beforeNodeRemoved(node)                                      | Called before a node is removed from the DOM                                             | return false to not remove the node                |
 | afterNodeRemoved(node)                                       | Called after a node is removed from the DOM                                              | none                                               |
-| beforeAttributeUpdated(attributeName, node, mutationType) | Called before an attribute on an element.  `mutationType` is either "updated" or "removed" | return false to not update or remove the attribute |
+| beforeAttributeUpdated(attributeName, node, mutationType)    | Called before an attribute on an element. `mutationType` is either "updated" or "removed"| return false to not update or remove the attribute |
+| beforeNodePantried(node)                                     | Called before moving the node into the pantry during the second pass (if enabled)        | return false to not move the node into the pantry  |
 
 ### The `head` tag
 
@@ -140,6 +142,15 @@ of the script.
 Similarly, you may wish to preserve an element even if it is not in `new`.  You can use the attribute `im-preserve='true'`
 in this case to retain the element.
 
+#### Two Pass Mode
+
+If the `twoPass` option is enabled, Idiomorph will try to maintain more state by temporarily moving some nodes into a
+hidden pantry div. After the initial morph is complete, it runs a second pass, moving the pantried nodes back into the
+document.  These are nodes that would otherwise have been removed and readded, losing identity and state. This is
+particularly useful for reordering morphs involving elements with non-attribute state, e.g. video elements, elements
+preserved with `data-turbo-permanent`, etc. Also, when this option is enabled, Idiomorph will use the upcoming
+`Element#moveBefore` API if it exists, falling back to `Element#insertBefore` if not.
+
 #### Additional Configuration
 
 You are also able to override these behaviors, see the `head` config object in the source code.

TESTING.md

@@ -47,6 +47,17 @@ npm run debug
 ```
 This will start the server, and open the test runner in a browser. From there you can choose a test file to run.
 
+## Forcing Two Pass Mode
+If the `DEFAULT_TWO_PASS` environment variable is set before running the tests, Idiomorph will default to two-pass mode. This is useful for running the entire test suite with two-pass on.
+
+## GitHub Actions CI matrix
+On each push and PR, GitHub Actions runs the following test matrix:
+
+1. Normal baseline `npm run ci` run
+2. With experimental moveBefore enabled in the browser
+3. With two-pass mode forced
+4. With both moveBefore enabled and two-pass mode forced
+
 ## Code Coverage Report
 After a test run completes, you can open `coverage/lcov-report/index.html` to view the code coverage report. On Ubuntu you can run:
 ```bash

src/idiomorph.js

@@ -20,6 +20,7 @@
  * @property {function(Element): boolean} [beforeNodeRemoved]
  * @property {function(Element): void} [afterNodeRemoved]
  * @property {function(string, Element, "update" | "remove"): boolean} [beforeAttributeUpdated]
+ * @property {function(Element): boolean} [beforeNodePantried]
  */
 
 /**
@@ -60,6 +61,7 @@
  * @property {(function(Node): boolean) | NoOp} beforeNodeRemoved
  * @property {(function(Node): void) | NoOp} afterNodeRemoved
  * @property {(function(string, Element, "update" | "remove"): boolean) | NoOp} beforeAttributeUpdated
+ * @property {(function(Node): boolean) | NoOp} beforeNodePantried
  */
 
 /**
@@ -70,6 +72,7 @@
  * @property {boolean} [ignoreActiveValue]
  * @property {ConfigCallbacksInternal} callbacks
  * @property {ConfigHeadInternal} head
+ * @property {boolean} [twoPass]
  */
 
 /**
@@ -78,7 +81,7 @@
  * @param {Element | Document} oldNode
  * @param {Element | Node | HTMLCollection | Node[] | string | null} newContent
  * @param {Config} [config]
- * @returns {undefined | HTMLCollection | Node[]}
+ * @returns {undefined | Node[]}
  */
 
 // base IIFE to define idiomorph
@@ -103,6 +106,7 @@ var Idiomorph = (function () {
          * @property {Set<string>} deadIds
          * @property {ConfigInternal['callbacks']} callbacks
          * @property {ConfigInternal['head']} head
+         * @property {HTMLDivElement} pantry
          */
 
         //=============================================================================
@@ -129,7 +133,7 @@ var Idiomorph = (function () {
                 beforeNodeRemoved: noOp,
                 afterNodeRemoved: noOp,
                 beforeAttributeUpdated: noOp,
-
+                beforeNodePantried: noOp,
             },
             head: {
                 style: 'merge',
@@ -153,7 +157,7 @@ var Idiomorph = (function () {
          * @param {Element | Document} oldNode
          * @param {Element | Node | HTMLCollection | Node[] | string | null} newContent
          * @param {Config} [config]
-         * @returns {undefined | HTMLCollection | Node[]}
+         * @returns {undefined | Node[]}
          */
         function morph(oldNode, newContent, config = {}) {
 
@@ -177,7 +181,7 @@ var Idiomorph = (function () {
          * @param {Element} oldNode
          * @param {Element} normalizedNewContent
          * @param {MorphContext} ctx
-         * @returns {undefined | HTMLCollection| Node[]}
+         * @returns {undefined | Node[]}
          */
         function morphNormalizedContent(oldNode, normalizedNewContent, ctx) {
             if (ctx.head.block) {
@@ -202,6 +206,9 @@ var Idiomorph = (function () {
 
                 // innerHTML, so we are only updating the children
                 morphChildren(normalizedNewContent, oldNode, ctx);
+                if (ctx.config.twoPass) {
+                    restoreFromPantry(oldNode, ctx);
+                }
                 return Array.from(oldNode.children);
 
             } else if (ctx.morphStyle === "outerHTML" || ctx.morphStyle == null) {
@@ -220,7 +227,11 @@ var Idiomorph = (function () {
                     // if there was a best match, merge the siblings in too and return the
                     // whole bunch
                     if (morphedNode) {
-                        return insertSiblings(previousSibling, morphedNode, nextSibling);
+                        const elements = insertSiblings(previousSibling, morphedNode, nextSibling);
+                        if (ctx.config.twoPass) {
+                            restoreFromPantry(morphedNode.parentNode, ctx);
+                        }
+                        return elements
                     }
                 } else {
                     // otherwise nothing was added to the DOM
@@ -337,10 +348,14 @@ var Idiomorph = (function () {
 
                 // if we are at the end of the exiting parent's children, just append
                 if (insertionPoint == null) {
-                    if (ctx.callbacks.beforeNodeAdded(newChild) === false) continue;
-
-                    oldParent.appendChild(newChild);
-                    ctx.callbacks.afterNodeAdded(newChild);
+                    // skip add callbacks when we're going to be restoring this from the pantry in the second pass
+                    if (ctx.config.twoPass && ctx.persistentIds.has(/** @type {Element} */ (newChild).id)) {
+                        oldParent.appendChild(newChild);
+                    } else {
+                        if (ctx.callbacks.beforeNodeAdded(newChild) === false) continue;
+                        oldParent.appendChild(newChild);
+                        ctx.callbacks.afterNodeAdded(newChild);
+                    }
                     removeIdsFromConsideration(ctx, newChild);
                     continue;
                 }
@@ -377,10 +392,15 @@ var Idiomorph = (function () {
 
                 // abandon all hope of morphing, just insert the new child before the insertion point
                 // and move on
-                if (ctx.callbacks.beforeNodeAdded(newChild) === false) continue;
 
-                oldParent.insertBefore(newChild, insertionPoint);
-                ctx.callbacks.afterNodeAdded(newChild);
+                // skip add callbacks when we're going to be restoring this from the pantry in the second pass
+                if (ctx.config.twoPass && ctx.persistentIds.has(/** @type {Element} */ (newChild).id)) {
+                    oldParent.insertBefore(newChild, insertionPoint);
+                } else {
+                    if (ctx.callbacks.beforeNodeAdded(newChild) === false) continue;
+                    oldParent.insertBefore(newChild, insertionPoint);
+                    ctx.callbacks.afterNodeAdded(newChild);
+                }
                 removeIdsFromConsideration(ctx, newChild);
             }
 
@@ -718,13 +738,21 @@ var Idiomorph = (function () {
                 ignoreActive: mergedConfig.ignoreActive,
                 ignoreActiveValue: mergedConfig.ignoreActiveValue,
                 idMap: createIdMap(oldNode, newContent),
-                persistentIds: persistentIdSet(oldNode, newContent),
                 deadIds: new Set(),
+                persistentIds: mergedConfig.twoPass ? createPersistentIds(oldNode, newContent) : new Set(),
+                pantry: mergedConfig.twoPass ? createPantry() : document.createElement("div"),
                 callbacks: mergedConfig.callbacks,
                 head: mergedConfig.head
             }
         }
 
+        function createPantry() {
+            const pantry = document.createElement("div");
+            pantry.hidden = true;
+            document.body.insertAdjacentElement("afterend", pantry);
+            return pantry;
+        }
+
         /**
          *
          * @param {Node | null} node1
@@ -1073,10 +1101,77 @@ var Idiomorph = (function () {
         //   places where tempNode may be just a Node, not an Element
         function removeNode(tempNode, ctx) {
             removeIdsFromConsideration(ctx, tempNode)
-            if (ctx.callbacks.beforeNodeRemoved(tempNode) === false) return;
+            // skip remove callbacks when we're going to be restoring this from the pantry in the second pass
+            if (ctx.config.twoPass && hasPersistentIdNodes(ctx, tempNode) && tempNode instanceof Element) {
+                moveToPantry(tempNode, ctx);
+            } else {
+                if (ctx.callbacks.beforeNodeRemoved(tempNode) === false) return;
+                tempNode.parentNode?.removeChild(tempNode);
+                ctx.callbacks.afterNodeRemoved(tempNode);
+            }
+        }
+
+        /**
+         *
+         * @param {Node} node
+         * @param {MorphContext} ctx
+         */
+        function moveToPantry(node, ctx) {
+            if (ctx.callbacks.beforeNodePantried(node) === false) return
+
+            Array.from(node.childNodes).forEach(child => {
+                moveToPantry(child, ctx);
+            });
+
+            // After processing children, process the current node
+            if (ctx.persistentIds.has(/** @type {Element} */ (node).id)) {
+                // @ts-ignore - use proposed moveBefore feature
+                if (ctx.pantry.moveBefore) {
+                    // @ts-ignore - use proposed moveBefore feature
+                    ctx.pantry.moveBefore(node, null);
+                } else {
+                    ctx.pantry.insertBefore(node, null);
+                }
+            } else {
+                if (ctx.callbacks.beforeNodeRemoved(node) === false) return;
+                node.parentNode?.removeChild(node);
+                ctx.callbacks.afterNodeRemoved(node);
+            }
+        }
 
-            tempNode.parentNode?.removeChild(tempNode);
-            ctx.callbacks.afterNodeRemoved(tempNode);
+        /**
+         *
+         * @param {Node | null} root
+         * @param {MorphContext} ctx
+         */
+        function restoreFromPantry(root, ctx) {
+            if (root instanceof Element) {
+                Array.from(ctx.pantry.children).reverse().forEach(element => {
+                    const matchElement = root.querySelector(`#${element.id}`);
+                    if (matchElement) {
+                        // @ts-ignore - use proposed moveBefore feature
+                        if (matchElement.parentElement?.moveBefore) {
+                            // @ts-ignore - use proposed moveBefore feature
+                            matchElement.parentElement.moveBefore(element, matchElement);
+                            while (matchElement.hasChildNodes()) {
+                                // @ts-ignore - use proposed moveBefore feature
+                                element.moveBefore(matchElement.firstChild, null);
+                            }
+                        } else {
+                            matchElement.before(element);
+                            while (matchElement.firstChild) {
+                                element.insertBefore(matchElement.firstChild, null)
+                            }
+                        }
+                        if (ctx.callbacks.beforeNodeMorphed(element, matchElement) !== false) {
+                            syncNodeFrom(matchElement, element, ctx);
+                            ctx.callbacks.afterNodeMorphed(element, matchElement);
+                        }
+                        matchElement.remove();
+                    }
+                });
+                ctx.pantry.remove();
+            }
         }
 
         //=============================================================================
@@ -1118,6 +1213,21 @@ var Idiomorph = (function () {
             }
         }
 
+        /**
+         *
+         * @param {MorphContext} ctx
+         * @param {Node} node
+         * @returns {boolean}
+         */
+        function hasPersistentIdNodes(ctx, node) {
+            for (const id of ctx.idMap.get(node) || EMPTY_SET) {
+                if (ctx.persistentIds.has(id)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
         /**
          *
          * @param {MorphContext} ctx
@@ -1139,15 +1249,15 @@ var Idiomorph = (function () {
         }
 
         /**
-         * @param {Element} Content
+         * @param {Element} content
          * @returns {Element[]}
          */
-        function nodesWithIds(Content) {
-            let Nodes = Array.from(Content.querySelectorAll('[id]'));
-            if(Content.id) {
-               Nodes.push(Content);
+        function nodesWithIds(content) {
+            let nodes = Array.from(content.querySelectorAll('[id]'));
+            if(content.id) {
+               nodes.push(content);
             }
-            return Nodes;
+            return nodes;
         }
 
         /**
@@ -1206,14 +1316,13 @@ var Idiomorph = (function () {
          * @param {Element} newContent  the new content to morph to
          * @returns {Set<string>} the id set of all persistent nodes that exist in both old and new content
          */
-        function persistentIdSet(oldContent, newContent) {
+        function createPersistentIds(oldContent, newContent) {
+            const toIdTagName = node => node.tagName+'#'+node.id;
+            const oldIdSet = new Set(nodesWithIds(oldContent).map(toIdTagName));
+
             let matchIdSet = new Set();
-            let oldIdSet = new Set();
-            for (const oldNode of nodesWithIds(oldContent)) {
-                oldIdSet.add(oldNode.id+':'+oldNode.tagName);
-            }
             for (const newNode of nodesWithIds(newContent)) {
-                if (oldIdSet.has(newNode.id+':'+newNode.tagName)) {
+                if (oldIdSet.has(toIdTagName(newNode))) {
                     matchIdSet.add(newNode.id);
                 }
             }

test/bootstrap.js

@@ -52,7 +52,7 @@ describe("Bootstrap test", function(){
         print(div1);
 
         // first paragraph should have been discarded in favor of later matches
-        d1.innerHTML.should.equal("A");
+        d1.innerHTML.should.not.equal("D");
 
         // second and third paragraph should have morphed
         d2.innerHTML.should.equal("E");