From 3a711790c5eb2867eeaae643c3a3efc905c7c239 Mon Sep 17 00:00:00 2001 From: Tim Perry Date: Wed, 23 Oct 2013 22:03:43 +0100 Subject: [PATCH 1/8] PoC saucelabs test runner --- saucelabs.js | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 saucelabs.js diff --git a/saucelabs.js b/saucelabs.js new file mode 100644 index 000000000..29a3a215d --- /dev/null +++ b/saucelabs.js @@ -0,0 +1,29 @@ +var port = 8081; + +// Create a web server for the local dir +var connect = require('connect'); +var server = connect.createServer( + connect.static(__dirname) +).listen(port); + +// Tell saucelabs to run some tests +var username = process.env['SAUCE_USERNAME']; +var accessKey = process.env['SAUCE_ACCESS_KEY']; + +var request = require('request'); + +request.post( + 'https://saucelabs.com/rest/v1/' + username + '/js-tests', + { + auth: { user: username, pass: accessKey }, + json: { + platforms: [[ "Windows 7", "chrome", "27" ]], + url: "http://localhost:" + port + "/test/index.html", + framework: "qunit" + } + }, + function (error, response, body) { + console.log(response.statusCode); + console.log(body); + } +); \ No newline at end of file From a717f5c03077e1a288e8595d3cb5fc92a6b7357b Mon Sep 17 00:00:00 2001 From: Tim Perry Date: Wed, 23 Oct 2013 23:10:26 +0100 Subject: [PATCH 2/8] Initial working saucelabs test result recording --- saucelabs.js | 46 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/saucelabs.js b/saucelabs.js index 29a3a215d..de0c3d7ee 100644 --- a/saucelabs.js +++ b/saucelabs.js @@ -7,6 +7,7 @@ var server = connect.createServer( ).listen(port); // Tell saucelabs to run some tests +var browser = JSON.parse(process.env['SAUCE_BROWSER']); var username = process.env['SAUCE_USERNAME']; var accessKey = process.env['SAUCE_ACCESS_KEY']; @@ -17,13 +18,50 @@ request.post( { auth: { user: username, pass: accessKey }, json: { - platforms: [[ "Windows 7", "chrome", "27" ]], + platforms: [ browser ], url: "http://localhost:" + port + "/test/index.html", framework: "qunit" } }, function (error, response, body) { - console.log(response.statusCode); - console.log(body); + if (response.statusCode == 200) { + var testIdentifier = body; + waitForTestCompletion(testIdentifier); + } else { + console.error("Failed to submit test to SauceLabs, status " + response.statusCode + ", body:\n" + JSON.stringify(body)); + process.exit(3); + } } -); \ No newline at end of file +); + +function waitForTestCompletion(testIdentifier) { + request.post( + 'https://saucelabs.com/rest/v1/' + username + '/js-tests/status', + { + auth: { user: username, pass: accessKey }, + json: testIdentifier + }, + function (error, response, body) { + if (response.statusCode == 200) { + console.log(JSON.stringify(body)); + if (body["completed"] == true) { + handleTestResults(body["js tests"]); + } else { + waitForTestCompletion(testIdentifier); + } + } else { + console.error("Failed to check test status on SauceLabs, status " + response.statusCode + ", body:\n" + JSON.stringify(body)); + process.exit(4); + } + } + ); +} + +function handleTestResults(results) { + var allTestsSuccessful = results.reduce(function (passedSoFar, result) { + return passedSoFar && !!result['passed'] + }, true); + + console.log(allTestsSuccessful ? "Test passed" : "Test failed"); + process.exit(allTestsSuccessful ? 0 : 1); +} \ No newline at end of file From 2dbfcbb378bfedcc74c8b84eb2a2da236aa1a3fb Mon Sep 17 00:00:00 2001 From: Tim Perry Date: Wed, 23 Oct 2013 23:22:26 +0100 Subject: [PATCH 3/8] Updated travis config to run the saucelabs tests for various browsers --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fc1b2ccac..464505c84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ node_js: - "0.10" env: global: - - BIN="node" BUILD=false MAKE=false OPTION="" + - BIN="node" BUILD=false MAKE=false OPTION="" SAUCELABS_BROWSER=false matrix: - BUILD="compat" - BUILD="modern" @@ -14,6 +14,7 @@ env: - BIN="phantomjs" BUILD="compat" - BIN="phantomjs" BUILD="legacy" - BIN="phantomjs" BUILD="mobile" + - SAUCELABS_BROWSER="['Windows 7', 'firefox', '20']" matrix: include: - node_js: "0.10" @@ -47,6 +48,7 @@ before_install: - "[ $BIN == 'rhino' ] && echo -e '#!/bin/sh\\njava -jar /opt/rhino-1.7R5/js.jar $@' | sudo tee /usr/local/bin/rhino && sudo chmod +x /usr/local/bin/rhino || true" - "[ $BIN == 'ringo' ] && wget http://ringojs.org/downloads/ringojs-0.9.zip && sudo unzip ringojs-0.9 -d /opt && rm ringojs-0.9.zip || true" - "[ $BIN == 'ringo' ] && sudo ln -s /opt/ringojs-0.9/bin/ringo /usr/local/bin/ringo && sudo chmod +x /usr/local/bin/ringo || true" + - "[ '$SAUCELABS_BROWSER' != false ] && npm install connect request || true" script: - "[ $BIN == 'istanbul' ] && $BIN cover ./test/test.js || true" - "[ $BUILD != false ] && [ $BUILD != 'compat' ] && MAKE=true || true" @@ -56,3 +58,4 @@ script: - "[ $BUILD != false ] && cd ./test || true" - "[ $BUILD == false ] && true || $BIN $OPTION ./test.js ../dist/lodash.$BUILD.js" - "[ $BUILD == false ] && true || $BIN $OPTION ./test.js ../dist/lodash.$BUILD.min.js" + - "[ '$SAUCELABS_BROWSER' != false ] && node ./saucelabs.js || true" From eb10245df1145e7aa4ff278d66404e0d2cedd7af Mon Sep 17 00:00:00 2001 From: pimterry Date: Sun, 27 Oct 2013 15:00:23 +0000 Subject: [PATCH 4/8] Moved sauce connect management to saucelabs.js, for manual control --- .travis.yml | 13 ++++++--- saucelabs.js | 77 ++++++++++++++++++++++++++++++++++------------------ 2 files changed, 59 insertions(+), 31 deletions(-) diff --git a/.travis.yml b/.travis.yml index 464505c84..50045f9d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,12 @@ node_js: - "0.10" env: global: - - BIN="node" BUILD=false MAKE=false OPTION="" SAUCELABS_BROWSER=false + - BIN="node" BUILD=false MAKE=false OPTION="" SAUCELABS=false + + # SauceLabs credentials: + - SAUCE_USERNAME="pimterry" + - secure: "KBETrs7sRrU3V8itVgLzBUVkzP9XXpAEZVjbd8icoNL9eVW3h4Kk5g/bOa06IAml0ThnMZztIwMKkplCIam6W/Cv+eIOiQT2kkG32lHFwbVXEkbq+rKj8XmpRmXBf/Th63V+u5GMVmQlYrIfdXju6cFfSEZ2uG8SHVg9hFyyf8c=" + matrix: - BUILD="compat" - BUILD="modern" @@ -14,7 +19,7 @@ env: - BIN="phantomjs" BUILD="compat" - BIN="phantomjs" BUILD="legacy" - BIN="phantomjs" BUILD="mobile" - - SAUCELABS_BROWSER="['Windows 7', 'firefox', '20']" + - SAUCELABS=true matrix: include: - node_js: "0.10" @@ -48,7 +53,7 @@ before_install: - "[ $BIN == 'rhino' ] && echo -e '#!/bin/sh\\njava -jar /opt/rhino-1.7R5/js.jar $@' | sudo tee /usr/local/bin/rhino && sudo chmod +x /usr/local/bin/rhino || true" - "[ $BIN == 'ringo' ] && wget http://ringojs.org/downloads/ringojs-0.9.zip && sudo unzip ringojs-0.9 -d /opt && rm ringojs-0.9.zip || true" - "[ $BIN == 'ringo' ] && sudo ln -s /opt/ringojs-0.9/bin/ringo /usr/local/bin/ringo && sudo chmod +x /usr/local/bin/ringo || true" - - "[ '$SAUCELABS_BROWSER' != false ] && npm install connect request || true" + - "[ $SAUCELABS != false ] && npm install connect request sauce-tunnel || true" script: - "[ $BIN == 'istanbul' ] && $BIN cover ./test/test.js || true" - "[ $BUILD != false ] && [ $BUILD != 'compat' ] && MAKE=true || true" @@ -58,4 +63,4 @@ script: - "[ $BUILD != false ] && cd ./test || true" - "[ $BUILD == false ] && true || $BIN $OPTION ./test.js ../dist/lodash.$BUILD.js" - "[ $BUILD == false ] && true || $BIN $OPTION ./test.js ../dist/lodash.$BUILD.min.js" - - "[ '$SAUCELABS_BROWSER' != false ] && node ./saucelabs.js || true" + - "[ $SAUCELABS == false ] && true || node ./saucelabs.js" diff --git a/saucelabs.js b/saucelabs.js index de0c3d7ee..41bcd16e9 100644 --- a/saucelabs.js +++ b/saucelabs.js @@ -1,38 +1,61 @@ +var connect = require('connect'); +var SauceTunnel = require('sauce-tunnel'); +var request = require('request'); + var port = 8081; +var username = process.env['SAUCE_USERNAME']; +var accessKey = process.env['SAUCE_ACCESS_KEY']; +var platforms = [ + ["Windows 7", "chrome", ""], + ["Windows 7", "firefox", "24"], + ["Windows 7", "internet explorer", "9"] +]; // Create a web server for the local dir -var connect = require('connect'); var server = connect.createServer( connect.static(__dirname) ).listen(port); -// Tell saucelabs to run some tests -var browser = JSON.parse(process.env['SAUCE_BROWSER']); -var username = process.env['SAUCE_USERNAME']; -var accessKey = process.env['SAUCE_ACCESS_KEY']; - -var request = require('request'); - -request.post( - 'https://saucelabs.com/rest/v1/' + username + '/js-tests', - { - auth: { user: username, pass: accessKey }, - json: { - platforms: [ browser ], - url: "http://localhost:" + port + "/test/index.html", - framework: "qunit" - } - }, - function (error, response, body) { - if (response.statusCode == 200) { - var testIdentifier = body; - waitForTestCompletion(testIdentifier); - } else { - console.error("Failed to submit test to SauceLabs, status " + response.statusCode + ", body:\n" + JSON.stringify(body)); - process.exit(3); - } +// Set up sauce connect so we can use this server from saucelabs +var tunnelTimeout = 10000; +var tunnel = new SauceTunnel(username, accessKey, null, true, tunnelTimeout); +console.log("Opening sauce connect tunnel..."); +tunnel.start(function (success) { + if (success) { + console.log("Sauce connect tunnel opened"); + runTests(); + } else { + console.error("Failed to open sauce connect tunnel") + process.exit(2); } -); +}); + +function runTests() { + var testDefinition = { + platforms: platforms, + url: "http://localhost:" + port + "/test/index.html", + framework: "qunit" + }; + + console.log("Starting saucelabs tests: " + JSON.stringify(testDefinition)); + + request.post( + 'https://saucelabs.com/rest/v1/' + username + '/js-tests', + { + auth: { user: username, pass: accessKey }, + json: testDefinition + }, + function (error, response, body) { + if (response.statusCode == 200) { + var testIdentifier = body; + waitForTestCompletion(testIdentifier); + } else { + console.error("Failed to submit test to SauceLabs, status " + response.statusCode + ", body:\n" + JSON.stringify(body)); + process.exit(3); + } + } + ); +} function waitForTestCompletion(testIdentifier) { request.post( From 3f4c92d739e8971d49e00b1e982f23a6b7656d14 Mon Sep 17 00:00:00 2001 From: pimterry Date: Sun, 27 Oct 2013 19:29:50 +0000 Subject: [PATCH 5/8] Add test listener to QUnit to report results for saucelabs --- test/index.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/index.html b/test/index.html index 255766ceb..5d312b25f 100644 --- a/test/index.html +++ b/test/index.html @@ -16,6 +16,13 @@
+