Automate testing in Edge and Safari (via Sauce Labs)

.travis.yml

@@ -1,4 +1,3 @@
----
 language: node_js
 node_js: "6"
 dist: trusty
@@ -12,7 +11,19 @@ env:
   - ESHOST_TARGET=chakra
   - ESHOST_TARGET=jsshell
   - ESHOST_TARGET=chrome
-  - ESHOST_TARGET=remote ESHOST_REMOTE_BROWSERNAME=firefox
+  - ESHOST_TARGET=remote:firefox
+  # ESHOST_REMOTE_WEBDRIVER_SERVER depends on the SAUCE_ACCESS_KEY environment
+  # variable. That variable is not available until *after* the TravisCI JWT
+  # "plugin" runs., so the WebDriver server URL must be defined within the
+  # "install" script.
+  - ESHOST_TARGET=remote:edge
+    SAUCE_USERNAME=jugglinmike
+    ESHOST_WEB_HOST=eshost.test
+    ESHOST_REMOTE_WEBDRIVER_SERVER=
+  - ESHOST_TARGET=remote:safari
+    SAUCE_USERNAME=jugglinmike
+    ESHOST_WEB_HOST=eshost.test
+    ESHOST_REMOTE_WEBDRIVER_SERVER=
 install: |
     export ESHOST_SKIP_D8=1
     export ESHOST_SKIP_JSC=1
@@ -82,18 +93,91 @@ install: |
       unzip -d chrome chromedriver_linux64.zip
       PATH=$PATH:$(pwd)/chrome
       unset ESHOST_SKIP_CHROME;
-    elif [[ "$ESHOST_TARGET" == "remote" ]]; then
+    elif [[ "$ESHOST_TARGET" == "remote:firefox" ]]; then
       install_firefox;
       wget http://selenium-release.storage.googleapis.com/3.4/selenium-server-standalone-3.4.0.jar
       PATH=$PATH:$(pwd)/firefox java -jar selenium-server-standalone-3.4.0.jar &> selenium-server.log &
 
+      export ESHOST_REMOTE_BROWSERNAME=firefox
+      export ESHOST_REMOTE_PLATFORM=ANY
+      export ESHOST_REMOTE_VERSION=
+      unset ESHOST_SKIP_REMOTE;
+    elif [[ "$ESHOST_TARGET" == "remote:edge" || "$ESHOST_TARGET" == "remote:safari" ]]; then
+      wget https://saucelabs.com/downloads/sc-4.4.8-linux.tar.gz
+      tar -xvf sc-4.4.8-linux.tar.gz
+
+      ready_file=sauce-connect-ready-$RANDOM
+
+      sc-4.4.8-linux/bin/sc \
+        --logfile=sauce-connect.log \
+        --tunnel-domains=eshost.test \
+        --tunnel-identifier=$TRAVIS_JOB_NUMBER \
+        --no-remove-colliding-tunnels \
+        --readyfile=$ready_file &
+      SC_PID="$!"
+
+      echo "Waiting for Sauce Connect 'ready' file..."
+      while [ ! -f $ready_file ] && ps -f $SC_PID >&/dev/null; do
+        sleep .5
+      done
+
+      if [ ! -f $ready_file ]; then
+        echo "Sauce Connect 'ready' file not created."
+        exit 1;
+      fi
+
+      echo "Sauce Connect 'ready' file created. Continuing installation."
+
+      if [[ "$ESHOST_TARGET" == "remote:edge" ]]; then
+        export ESHOST_REMOTE_BROWSERNAME=MicrosoftEdge
+        export ESHOST_REMOTE_PLATFORM='Windows 10'
+        export ESHOST_REMOTE_VERSION=15.15063
+      elif [[ "$ESHOST_TARGET" == "remote:safari" ]]; then
+        export ESHOST_REMOTE_BROWSERNAME=safari
+        export ESHOST_REMOTE_PLATFORM='macOS 10.12'
+        export ESHOST_REMOTE_VERSION=10.0
+      fi
+
+      export ESHOST_REMOTE_WEBDRIVER_SERVER=https://$SAUCE_USERNAME:[email protected]/wd/hub
       unset ESHOST_SKIP_REMOTE;
     else
       exit 1;
     fi
 
     npm install
 after_script: |
-    if [[ "$ESHOST_TARGET" == "remote" ]]; then
-      cat selenium-server.log
-    fi
+  if [[ "$ESHOST_TARGET" == "remote:firefox" ]]; then
+    cat selenium-server.log
+  fi
+
+  if [[ "$SC_PID" != "" ]]; then
+    echo "Tearing down Sauce Connect tunnel."
+
+    kill $SC_PID
+
+    for i in 0 1 2 3 4 5 6 7 8 9 ; do
+      if kill -0 $SC_PID &>/dev/null ; then
+        echo "Waiting for Sauce Connect tunnel to exit..."
+        sleep 1
+      else
+        echo "Sauce Connect shutdown complete."
+        break
+      fi
+    done
+  fi
+
+  if [ -f sauce-connect.log ]; then
+    cat sauce-connect.log
+  fi
+addons:
+  hosts:
+    - eshost.test
+  # The "jwt" configuration was inserted by the `travis` executable in
+  # response to the following command:
+  #
+  #     travis encrypt  --add addons.jwt SAUCE_ACCESS_KEY=VALUE
+  #
+  # ...where `VALUE` is the actual access key for the repository owner as
+  # provided by Sauce Labs.
+  jwt:
+    secure: LgoC+2N9gim6lQErlFDM7ALqow1n9EMb3IDLWrBJuUcvedt8RSCZs3oaJJsa4Dz9CAFi9wYLb77fwuqaROQeg1M1EAp20XSaZYCVEZ99AAV+qQJcyzE+ksnsWrDVZr+07XB1MEIClUZn3ZZCoXUUojgMuM/seReKl2bUCVI9LVs3lrtAShFUO7Ovi0L35olfu8LIW4rWltH7oI2I1JoJkA/sXaQDCsOxtegng1W8vptdAb8CFg1mKQMHzMVxx85L50LNk6feXeT9yktfvVCCAqGZqkr89J0Z0YTKX0/NjS1GXyoEUvBEr9CUXF07e5qFhGrvg7BR3QvKUiu86BfHUERzkReUkQ7G76YW3ZlPvJbY9zWdnk4LQpmlsi1KHqq21CZ8wwxcX1MaYHPiRZTvF5+US3VBggJrzkQEUzuxsqmWuVqE655aVct1kbOlKqGgJ8OwUugel3rz90JAEOG4fJD+YpHraxP2rRUFhe7Y1SAj/L5+V168XIx8wrmAKS5X6zmX7AU+hcNknW4gmsPPv5TH3iNY4Nw2gpUx0Er7LS9xywcw2HUyZzFQ7vQcajGyC8yomwnBI2le7Pa9c2ZjoBSWlELZ/ldg/+0VpEEMQz7YLZZNP7f1+4+QuZI6AGMD4EiNn0JH0wFXWHkHjivhS/rqPSuaDsOdABKoX03YyXQ=

README.md

@@ -63,6 +63,19 @@ Gets an instance of a runner for a particular host type. See the table above for
   * **capabilities.platform**
   * **capabilities.version**
 * **webdriverServer**: for `remote` host only; URL of the WebDriver server to which commands should be issued
+* **noKeepAlive**: for `remote` host only; controls whether a WebDriver command is sent with each script evaluation; defaults to `false`
+
+##### "Keep Alive" signal
+
+Some WebDriver service providers consider long-running sessions with no
+WebDriver traffic as "timed out," and they may subsequently destroy such
+sessions. Because `eshost` circumvents the WebDriver protocol to execute
+scripts (using a separate WebSocket channel rather than the WebDriver "Execute
+Script" command) sessions that are actually active may be interpreted as "timed
+out." To avoid this, `eshost`'s WebDriver agent sends a "Get URL" WebDriver command
+prior to each script execution and discards the result.
+
+This behavior can be disabled via the `noKeepAlive` option.
 
 ### Agent API
 #### initialize(): Promise<void>

lib/agents/remote.js

@@ -40,6 +40,16 @@ class RemoteAgent extends WebDriverAgent {
     if (typeof options.hostPath === 'string') {
       throw new UnspecifiedOptionError('hostPath');
     }
+
+    this.noKeepAlive = !!options.noKeepAlive;
+  }
+
+  evalScript(...args) {
+    const firstOp = this.noKeepAlive ?
+      Promise.resolve() : this._driver.getCurrentUrl();
+
+    return firstOp
+      .then(() => super.evalScript(...args));
   }
 
   _createDriver() {

runtimes/browser.js

@@ -19,13 +19,14 @@ var $ = window.$ = {
     document.body.appendChild(frame);
     var fwin = frame.contentWindow;
     var fdoc = fwin.document;
-    var fscript = fdoc.createElement('script');
 
     // The following is a workaround for a bug in Chromium related to reporting
     // errors produced from evaluating code using `eval`.
     // https://bugs.chromium.org/p/chromium/issues/detail?id=746564
     fdoc.write('<body>');
 
+    var fscript = fdoc.createElement('script');
+
     fscript.textContent = this.source;
     fdoc.body.appendChild(fscript);
     var f$ = fwin.$;
@@ -58,17 +59,23 @@ var $ = window.$ = {
       if (!err) {
         // make up some error for Edge.
         err = {
-          name: 'Error',
+          name: 'UnknownESHostError',
           message: msg
         };
       }
 
       error = err;
     }
     document.body.appendChild(s);
-    if (window) {
+
+    /**
+     * Microsoft Edge throws a TypeError (message: "Object expected") when
+     * referencing the `window` identifier in an iframe that is not attached to
+     * some parent document.
+     */
+    try {
       window.onerror = null;
-    }
+    } catch (err) {}
 
     if (error) {
       return { type: 'throw', value: error };

test/runify.js

@@ -6,10 +6,15 @@ const assert = require('assert');
 const isWindows = process.platform === 'win32' ||
   process.env.OSTYPE === 'cygwin' ||
   process.env.OSTYPE === 'msys';
-const remoteCapabilities = {
-  browserName: process.env.ESHOST_REMOTE_BROWSERNAME || 'firefox',
-  platform: process.env.ESHOST_REMOTE_PLATFORM || 'ANY',
-  version: process.env.ESHOST_REMOTE_VERSION || ''
+const remoteOptions = {
+  webHost: process.env.ESHOST_WEB_HOST || 'localhost',
+  webdriverServer: process.env.ESHOST_REMOTE_WEBDRIVER_SERVER || 'http://localhost:4444/wd/hub',
+  capabilities: {
+    browserName: process.env.ESHOST_REMOTE_BROWSERNAME || 'firefox',
+    platform: process.env.ESHOST_REMOTE_PLATFORM || 'ANY',
+    version: process.env.ESHOST_REMOTE_VERSION || '',
+    'tunnel-identifier': process.env.TRAVIS_JOB_NUMBER
+  }
 };
 
 const hosts = [
@@ -20,11 +25,7 @@ const hosts = [
   ['jsc', { hostPath: 'jsc' }],
   ['chrome', { hostPath: 'chrome' }],
   ['firefox', { hostPath: 'firefox' }],
-  ['remote', {
-      webdriverServer: 'http://localhost:4444/wd/hub',
-      capabilities: remoteCapabilities
-    }
-  ],
+  ['remote', remoteOptions],
 ];
 
 const timeout = function(ms) {
@@ -41,19 +42,17 @@ hosts.forEach(function (record) {
   if (options.hostPath && isWindows) {
     options.hostPath += '.exe';
   }
+  const uncaughtErrorName = (effectiveType === 'MicrosoftEdge') ?
+    () => 'UnknownESHostError' : (name) => name;
 
   describe(`${type} (${options.hostPath || effectiveType})`, function () {
-    this.timeout(20000);
+    this.timeout((type === 'remote') ? 60000 : 20000);
 
     before(function() {
       if (process.env['ESHOST_SKIP_' + type.toUpperCase()]) {
         this.skip();
         return;
       }
-
-      if (type === 'remote') {
-        this.timeout(60 * 1000);
-      }
     });
 
     describe('normal script evaluation', function() {
@@ -71,7 +70,7 @@ hosts.forEach(function (record) {
       it('runs SyntaxErrors', function () {
         return agent.evalScript('foo x++').then(function (result) {
           assert(result.error, 'error is present');
-          assert.equal(result.error.name, 'SyntaxError');
+          assert.equal(result.error.name, uncaughtErrorName('SyntaxError'));
           assert.equal(result.stdout, '', 'stdout not present');
         });
       });
@@ -216,11 +215,14 @@ hosts.forEach(function (record) {
       });
 
       it('returns errors from evaling in new script', function () {
+        var expectedPattern = '^' + uncaughtErrorName('SyntaxError') + '\r?\n';
+        var expectedRe = new RegExp(expectedPattern, 'm');
+
         return agent.evalScript(`
           var completion = $.evalScript("x+++");
           print(completion.value.name);
         `).then(function(result) {
-          assert(result.stdout.match(/^SyntaxError\r?\n/m), 'Unexpected stdout: ' + result.stdout + result.stderr);
+          assert(result.stdout.match(expectedRe), 'Unexpected stdout: ' + result.stdout + result.stderr);
         });
       });
 
@@ -429,7 +431,7 @@ hosts.forEach(function (record) {
         // The GeckoDriver project cannot currently destroy browsing sessions
         // whose main thread is blocked.
         // https://github.com/mozilla/geckodriver/issues/825
-        if (effectiveType === 'firefox') {
+        if (['firefox', 'MicrosoftEdge', 'safari'].indexOf(effectiveType) > -1) {
           this.skip();
           return;
         }