Adding Acceptance Tests for compose based tests

.github/workflows/acceptance_tests.yaml

@@ -0,0 +1,23 @@
+name: Acceptance Tests
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+jobs:
+  acceptance-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: run tests
+        run: acceptance-tests/run
+      - name: Upload test results
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: tests-result
+          path: ats/result

acceptance-tests/.gitignore

@@ -0,0 +1,2 @@
+rseult
+assets

acceptance-tests/README.md

@@ -0,0 +1,73 @@
+# Acceptance Test Procedure
+
+TL:DR; From the project's root `./acceptance-tests/run`
+
+This directory contains Acceptance Test Procedure AKA black box tests.
+The tests are running over a lab managed by docker-compose and using playwright
+for browser automation.
+
+The `acceptance-tests` directory includes a special directory `infra` with the infrastructure
+required by the tests.
+For example, there's a `./infra/pion` directory with a Dockerfile and an SSH
+config file.
+
+Unlike production containers, lab containers' entry point
+often includes setup code usually found in the Dockerfile.
+This is done for flexibility and speed as we want the lab to use the latest source
+
+The script support some old style options, use `./acceptance-tests/run -h` to see the 
+all the options. It also accepts one of more argument with a directory name.
+
+## The setup
+
+We use [playwright](https://playwright.dev) as the test runner.
+In addition to browser automation the runner uses SSH to control the services.
+Thanks to compose, a name of a service is also its address so 
+
+The runner supports one environment variable - `PWOPTS` - one can use to pass
+options to playwright. The default is `-x` stopping the test on the first
+failure. It's rigged this way becuase ATPs are usually complex scenarios.
+Unlike unit tests, where each test function is independent, here each functions
+is a step in one test procedure. Once a step failed, `-x` makes playwright ignore
+the rest of the file.
+
+To get help on playwright options run:
+
+```bash
+docker compose -f acceptance-tests/data-channels/lab.yaml --project-directory . run --entrypoint "npx playwright test --help" runner
+```
+
+The above command uses the lab from the data-channels tests to bring up a runner
+and override its entrypoint. Instead of running the specs, get help on playwright.
+We use `--project-directory .` in all the tests to ensure the relative paths are
+relative to the project's root. 
+
+## Adding a test
+
+To create the `fubar` test, create a new directory in `./acceptance-tests/fubar` and start working
+one your `lab.yaml`. If your tests include a browser client your runner is
+best using the image from `infra/playwright`:
+
+```yaml
+version: "3.9"
+services:
+  runner:
+    build: 
+      context: .
+      dockerfile: ./acceptance-tests/infra/playwright/Dockerfile
+    volumes:
+      - ./acceptance-tests/fubar:/specs
+      - ./acceptance-tests/result:/result
+    environment:
+      PWOPTS: ${PWOPTS--x}
+```
+
+Not that we are mapping `/specs` to `acceptance-tests/fubar` as the first is where the image looks
+for specs. Once you've added your specs file to `./acceptance-tests/fubar` it will run whener you 
+`./acceptance-tests/run`. In fact, all a sub-directory of acceptance-tests needs is a `lab.yaml` file and the
+run script will try to bring it up.
+
+If you want to run just fubar you can use `./acceptance-tests/run acceptance-tests/fubar`.
+To run a clean build of fubar, ignoring all cache, use `-z`.
+
+

acceptance-tests/data-channels/example.spec.ts

@@ -0,0 +1,125 @@
+import { Buffer } from 'node:buffer';
+import { test, expect, Page, BrowserContext } from '@playwright/test'
+import { Client } from 'ssh2'
+
+
+test.describe("pion's data channels example", ()  => {
+
+    const sleep = (ms) => { return new Promise(r => setTimeout(r, ms)) }
+
+    let page: Page
+    let context: BrowserContext
+    let SSHconn: Client
+    let stream
+
+    test.beforeAll(async ({ browser }) => {
+        context = await browser.newContext()
+        page = await context.newPage()
+        page.on('console', (msg) => console.log('console log:', msg.text()))
+        page.on('pageerror', (err: Error) => console.log('PAGEERROR', err.message))
+        // Load the javascript file
+        page.on('load', () => page.evaluate(() => {
+                var newScript = document.createElement('script')
+                newScript.src = 'demo.js'
+                document.head.appendChild(newScript)
+            })
+        )
+        const response = await page.goto("http://client/demo.html")
+        await expect(response.ok()).toBeTruthy()
+        SSHconn = null
+    })
+
+    test('setup SSH', async () => {
+        while (SSHconn == null) {
+            try {
+                SSHconn = await new Promise((resolve, reject) => {
+                    const SSHconn = new Client()
+                    SSHconn.on('error', e => reject(e))
+                    SSHconn.on('ready', () => resolve(SSHconn))
+                    SSHconn.connect({
+                      host: 'pion',
+                      port: 22,
+                      username: 'pion',
+                      password: 'pion'
+                    })
+                })
+            } catch(e) {
+                console.log("SSH connection failed, retrying")
+                await sleep(3000)
+            }
+        }
+        // log key SSH events
+        SSHconn.on('error', e => console.log("ssh error", e))
+        SSHconn.on('close', e => {
+            console.log("ssh closed", e)
+        })
+        SSHconn.on('end', e => console.log("ssh ended", e))
+        SSHconn.on('keyboard-interactive', e => console.log("ssh interaction", e))
+    })
+    test('open the command stream', async () => {
+        let offer
+        while (!offer) {
+             await sleep(200)
+             offer = await page.evaluate(() =>
+                document.getElementById('localSessionDescription').value
+            )
+        }
+        try {
+            stream = await new Promise((resolve, reject) => {
+                const path = "/go/src/github.com/pion/webrtc/acceptance-tests/data-channels/start_server.bash"
+                SSHconn.exec(`bash ${path} ${offer}`,
+                        { pty: true }, async (err, s) => {
+                    if (err)
+                        reject(err)
+                    else 
+                        resolve(s)
+                })
+            })
+        } catch(e) { expect(e).toBeNull() }
+        stream.on('close', (code, signal) => {
+            console.log(`SSH closed with ${signal}`)
+            SSHconn.end()
+        }) 
+    })
+    test('transmit and receive data', async()=> {
+        let eof = false
+        let lineCounter = 0
+        stream.on('data', lines => 
+            new Buffer.from(lines).toString().split("\r\n")
+                .forEach(async (line: string) => {
+                if (!line)
+                    return
+                lineCounter++
+                if (lineCounter == 1) {
+                    // copy the answer to the page
+                    await page.evaluate(async (answer) =>
+                        document.getElementById("remoteSessionDescription")
+                                .value = answer,
+                        line)
+                    page.locator("data-test-id=start-session").click()
+                    // set the message to EOF
+                    await page.evaluate(async () => 
+                        document.getElementById("message").value = "EOF")
+                    // wait for the send channel to open
+                    let connected = false
+                    while (!connected) {
+                        await sleep(200)
+                        connected = await page.evaluate(() => sendChannel.readyState == "open")
+                    }
+                    // send the message
+                    await page.locator("data-test-id=send-message").click()
+                    return
+                }
+                // exit the test when EOF was received from the server
+                if (line.includes("EOF"))
+                    eof = true
+            })
+        ).stderr.on('data', (data) => {
+              console.log("ERROR: " + data)
+        })
+        // wait for the EOF message
+        while (!eof) {
+            await sleep(200)
+        }
+    })
+})

acceptance-tests/data-channels/lab.yaml

@@ -0,0 +1,30 @@
+version: "3.9"
+services:
+  runner:
+    build: 
+      context: .
+      dockerfile: ./acceptance-tests/infra/playwright/Dockerfile
+    volumes:
+      - ./acceptance-tests/data-channels:/specs
+      - ./acceptance-tests/result:/result
+    depends_on:
+      - client
+      - pion
+    environment:
+      PWOPTS: ${PWOPTS--x}
+  client:
+    image: halverneus/static-file-server:latest
+    environment:
+      PORT: 80
+    expose:
+      - "80"
+    volumes:
+      - ./examples/data-channels/jsfiddle:/web
+  pion:
+    build:
+      context: .
+      dockerfile: ./acceptance-tests/infra/pion/Dockerfile
+    expose:
+      - "22"
+    volumes:
+      - .:/go/src/github.com/pion/webrtc

acceptance-tests/data-channels/start_server.bash

@@ -0,0 +1,14 @@
+#!/bin/bash
+GO=/usr/local/go/bin/go
+cd "/go/src/github.com/pion/webrtc/examples/data-channels"
+TMP=`mktemp`
+$GO build -buildvcs=false -o $HOME/datachannels . > $TMP 2>&1
+
+if [ $? -eq 0 ]; then
+    echo $1 | $HOME/datachannels
+else
+    # on error send the last 5 lines of output as a single line
+    # so it's displayed in the browser
+    tail -5 $TMP | tr '\n' ':'
+fi
+rm $TMP

e2e/Dockerfile → acceptance-tests/e2e/Dockerfile

@@ -6,7 +6,6 @@ RUN apk add --no-cache \
 
 ENV CGO_ENABLED=0
 
-COPY . /go/src/github.com/pion/webrtc
-WORKDIR /go/src/github.com/pion/webrtc/e2e
+WORKDIR /go/src/github.com/pion/webrtc/acceptance-tests/e2e
 
 CMD ["go", "test", "-tags=e2e", "-v", "."]

acceptance-tests/e2e/lab.yaml

@@ -0,0 +1,8 @@
+version: "3.9"
+services:
+  runner:
+    build: 
+      context: .
+      dockerfile: ./acceptance-tests/e2e/Dockerfile
+    volumes:
+      - .:/go/src/github.com/pion/webrtc

acceptance-tests/infra/pion/Dockerfile

@@ -0,0 +1,9 @@
+FROM golang:bullseye
+RUN apt-get update
+RUN apt-get install -y git bash openssh-server
+COPY ./acceptance-tests/infra/pion/ssh_config /etc/ssh/
+# pion user with pion password to be used by clients' ssh connections
+RUN useradd -s /bin/bash -d /home/pion -M -p '$y$j9T$6LsNr6MtK4Nt6NeECphjP1$Try0q9dVUdZGihzzxJZ0soO09wQhseYzyB/E2Jf4tO8' pion
+RUN mkdir /home/pion
+RUN chown pion /home/pion
+CMD /etc/init.d/ssh start && tail -f /dev/null

acceptance-tests/infra/pion/ssh_config

@@ -0,0 +1,9 @@
+Include /etc/ssh/ssh_config.d/*.conf
+Host *
+PasswordAuthentication yes
+SendEnv LANG LC_*
+HashKnownHosts yes
+GSSAPIAuthentication yes
+PermitEmptyPasswords yes
+PermitRootLogin yes
+

acceptance-tests/infra/playwright/Dockerfile

@@ -0,0 +1,9 @@
+FROM mcr.microsoft.com/playwright:v1.26.0-focal 
+ENV NODE_PATH="/usr/lib/node_modules"
+RUN mkdir /runner
+WORKDIR /runner
+COPY ./acceptance-tests/infra/playwright/* .
+RUN yarn install --frozen-lockfile
+# /specs is set by lab.yaml, here copy the specs on run
+# to make sure we're running the latest version
+CMD cp /specs/*.spec.ts . && npx playwright test ${PWOPTS}

acceptance-tests/infra/playwright/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "pion-playwright",
+  "version": "0.1.0",
+  "private": true,
+  "description": "A playwright client for pion testing",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@playwright/test": "^1.22.2",
+    "playwright-chromium": "^1.22.2",
+    "playwright-firefox": "^1.22.2",
+    "playwright-webkit": "^1.22.2",
+    "redis": "<5",
+    "ssh2": "~1.11.0",
+    "wait-port": "^0.2.9",
+    "@types/node": "^18.7.18"
+  }
+}