Merge pull request pytest-dev#8 from manahl/features/devpi-server

Features/devpi server

.gitignore

@@ -1,4 +1,5 @@
 *.py[cod]
+.cache
 
 # C extensions
 *.so

CHANGES.md

@@ -1,6 +1,13 @@
 
 ## Changelog
 
+### 1.1.0 (2016-2-15)
+
+ * New plugin: devpi server fixture
+ * Changed default behavior of workspace.run() to not use a subshell for security reasons
+ * Corrected virtualenv.run() method to handle arguments the same as the parent method workspace.run()
+ * Removed deprecated '--distribute' from virtualenv args
+
 ### 1.0.1 (2015-12-23)
 
  *  Packaging bugfix

Makefile

@@ -3,6 +3,7 @@ PACKAGES=pytest-fixture-config          \
          pytest-shutil                  \
          pytest-server-fixtures         \
          pytest-pyramid-server          \
+         pytest-devpi-server            \
          pytest-listener                \
          pytest-qt-app                  \
          pytest-svn                     \
@@ -26,8 +27,7 @@ COPY_FILES=VERSION CHANGES.md common_setup.py MANIFEST.in
 DIST_FORMATS=sdist bdist_wheel bdist_egg
 UPLOAD_OPTS=
 
-
-.PHONY: venv copyfiles develop test dist upload clean
+.PHONY: venv copyfiles install test dist upload clean
 
 
 $(VENV_PYTHON):
@@ -47,19 +47,21 @@ copyfiles:
 	    cd ..;                                          \
     done
 
-develop: venv copyfiles
+install: venv copyfiles
 	for package in $(PACKAGES); do                      \
 	    cd $$package;                                   \
-	    ../$(VENV_PYTHON) setup.py develop || exit 1;   \
+	    ../$(VENV_PYTHON) setup.py bdist_egg || exit 1; \
+	    ../venv/bin/easy_install dist/*.egg || exit 1;  \
 	    cd ..;                                          \
     done
 
-test: develop
+test: install
 	for package in $(PACKAGES); do                      \
 	    (cd $$package;                                  \
-	     ../$(VENV_PYTHON) setup.py test;               \
+	     ../$(VENV_PYTHON) setup.py test -sv -ra || touch ../FAILED; \
 	    )                                               \
-    done
+    done;                                               \
+    [ -f FAILED ] && exit 1  || true
 
 dist: venv copyfiles
 	for package in $(PACKAGES); do                     \
@@ -88,6 +90,7 @@ clean:
 	done;                                                     \
 	rm -rf venv pytest-pyramid-server/vx pip-log.txt
 	find . -name *.pyc -delete
+	rm -f FAILED
 
 all: 
-	test
+	test

circle.yml

@@ -7,7 +7,7 @@ dependencies:
 
 test:
   override:
-    - make venv
+    - make clean venv
     # Just pull in PyQt4 from the system site-packages - if we create the venv with --system-site-packages
     # then we can't install many of the other dependencies due to install errors in pip/setuptools
     - cd venv/lib/python2.7/site-packages; ln -s $(python2.7 -c "import PyQt4; print PyQt4.__path__[0]"); ln -s $(python2.7 -c "import sip; print sip.__file__");

common_setup.py

@@ -62,4 +62,6 @@ def common_setup(src_dir):
             license='MIT license',
             platforms=['unix', 'linux'],
             cmdclass={'test': PyTest},
+            setup_requires=['setuptools-git'],
+            include_package_data=True
             )

pytest-devpi-server/README.md

@@ -0,0 +1,82 @@
+# Py.test DevPi Server Fixture
+
+DevPi server fixture for ``py.test``. The server is session-scoped by default 
+and run in a subprocess and temp dir to cleanup when it's done. 
+
+After the server has started up it will create a single user with a password, 
+and an index for that user. It then activates that index and provides a
+handle to the ``devpi-client`` API so you can manipulate the server in your tests.
+
+## Installation
+
+Install using your favourite package manager:
+
+```bash
+    pip install pytest-devpi-server
+    #  or..
+    easy_install pytest-devpi-server
+```
+
+Enable the fixture explicitly in your tests or conftest.py (not required when using setuptools entry points):
+
+```python
+    pytest_plugins = ['pytest_devpi_server']
+```
+                      
+## Example 
+
+Here's a noddy test case showing the main functionality:
+
+```python
+    def test_devpi_server(devpi_server):
+        # This is the client API for the server that's bound directly to the 'devpi' command-line tool.
+        # Here we list the available indexes
+        print devpi_server.api('use', '-l')
+
+        # Create and use another index
+        devpi_server.api('index', '-c', 'myindex')
+        devpi_server.api('index', 'use', 'myindex')
+
+        # Upload a package 
+        import os
+        os.chdir('/path/to/my/setup/dot/py')
+        devpi_server.api('upload')
+
+        # Get some json data
+        import json
+        res = devpi_server.api('getjson', '/user/myindex')
+        assert json.loads(res)['result']['projects'] == ['my-package-name']
+
+```        
+        
+## `DevpiServer` class
+
+Using this with the default `devpi_server` py.test fixture is good enough for a lot of 
+use-cases however you may wish to have more fine-grained control about the server configuration.
+
+To do this you can use the underlying server class directly - this is an implenentation of the
+`pytest-server-fixture` framework and as such acts as a context manager:
+
+```python
+    import json
+    from pytest_devpi_server import DevpiServer
+
+    def test_custom_server():
+        with DevPiServer(
+              # You can specify you own initial user and index
+              user='bob',
+              password='secret',
+              index='myindex',
+
+              # You can provide a zip file that contains the initial server database, 
+              # this is useful to pre-load any required packages for a test run
+              data='/path/to/data.zip'
+           ) as server:
+
+               assert not server.dead
+               res = server.api('getjson', '/bob/myindex')
+               assert 'pre-loaded-package' in json.loads(res)['result']['projects'] 
+
+        # Server should now be dead
+        assert server.dead   
+```

pytest-devpi-server/pytest_devpi_server/__init__.py

@@ -0,0 +1,124 @@
+'''
+Created on 25 Apr 2012
+
+@author: eeaston
+'''
+import os
+import sys
+import zipfile
+import logging
+import cStringIO
+
+import pkg_resources
+from pytest import yield_fixture
+from path import path
+import devpi_server as _devpi_server
+from devpi.main import main as devpi_client
+from pytest_server_fixtures.http import HTTPTestServer
+
+log = logging.getLogger(__name__)
+
+@yield_fixture(scope='session')
+def devpi_server(request):
+    """ Session-scoped Devpi server run in a subprocess, out of a temp dir. 
+        Out-of-the-box it creates a single user an index for that user, then
+        uses that index.
+        
+        Methods
+        -------
+        api():    Client API method, directly bound to the devpi-client command-line tool.  Examples:   
+        ...          api('index', '-c', 'myindex') to create an index called 'myindex'
+        ...          api('getjson', '/user/myindex') to return the json string describing this index
+        
+        Attributes
+        ----------
+        uri:          Server URI
+        user:         Initially created username
+        password:     Initially created password
+        index:        Initially created index name
+        server_dir:   Path to server database
+        client_dir:   Path to client directory
+           
+        .. also inherits all attributes from the `workspace` fixture 
+        
+        For more fine-grained control over these attributes, use the class directly and pass in
+        constructor arguments.
+    """
+    with DevpiServer() as server:
+        server.start()
+        yield server
+
+
+class DevpiServer(HTTPTestServer):
+
+    def __init__(self, offline=True, debug=False, data=None, user="testuser", password="", index='dev', **kwargs):
+        """ Devpi Server instance.
+
+        Parameters
+        ----------
+        offline :  `bool`
+            Run in offline mode. Defaults to True
+        data:  `str`
+           Filesystem path to a zipfile archive of the initial server data directory.
+           If not set and in offline mode, it uses a pre-canned snapshot of a 
+           newly-created empty server.
+        """
+        self.debug = debug
+        if os.getenv('DEBUG') in (True, '1', 'Y', 'y'):
+            self.debug = True
+        super(DevpiServer, self).__init__(**kwargs)
+
+        self.offline = offline
+        self.data = data
+        self.server_dir = self.workspace / 'server'
+        self.client_dir = self.workspace / 'client'
+        self.user = user
+        self.password = password
+        self.index = index
+
+    @property
+    def run_cmd(self):
+        res = [path(sys.exec_prefix) / 'bin' / 'python',
+                path(sys.exec_prefix) / 'bin' / 'devpi-server',
+                '--serverdir', self.server_dir,
+                '--host', self.hostname,
+                '--port', str(self.port)
+                ]
+        if self.offline:
+            res.append('--offline-mode')
+        if self.debug:
+            res.append('--debug')
+        return res
+
+    def api(self, *args):
+        """ Client API.
+        """
+        client_args = ['devpi']
+        client_args.extend(args)
+        client_args.extend(['--clientdir', self.client_dir])
+        log.info(' '.join(client_args))
+        captured = cStringIO.StringIO()
+        stdout = sys.stdout
+        sys.stdout = captured
+        try:
+            devpi_client(client_args)
+            return captured.getvalue()
+        finally:
+            sys.stdout = stdout
+
+
+    def pre_setup(self):
+        if self.data:
+            log.info("Extracting initial server data from {}".format(self.data))
+            zipfile.ZipFile(self.data, 'r').extractall(self.server_dir)
+
+
+    def post_setup(self):
+        # Connect to our server
+        self.api('use', self.uri)
+        # Create and log in initial user
+        self.api('user', '-c', self.user, 'password={}'.format(self.password))
+        self.api('login', self.user, '--password={}'.format(self.password))
+        # Create and use stand-alone index
+        self.api('index', '-c', self.index, 'bases=')
+        self.api('use', self.index)

pytest-devpi-server/setup.cfg

@@ -0,0 +1,8 @@
+[pytest]
+# This section sets configuration for all invocations of py.test,
+# both standalone cmdline and running via setup.py
+norecursedirs = 
+	.git 
+	*.egg 
+	build 
+	dist

pytest-devpi-server/setup.py

@@ -0,0 +1,46 @@
+from setuptools import setup, find_packages
+from common_setup import common_setup
+
+classifiers = [
+    'License :: OSI Approved :: MIT License',
+    'Development Status :: 5 - Production/Stable',
+    'Topic :: Software Development :: Libraries',
+    'Topic :: Software Development :: Testing',
+    'Topic :: Utilities',
+    'Intended Audience :: Developers',
+    'Operating System :: POSIX',
+    'Framework :: Pyramid',
+    'Programming Language :: Python :: 2',
+    'Programming Language :: Python :: 2.6',
+    'Programming Language :: Python :: 2.7',
+]
+
+install_requires = ['pytest-server-fixtures',
+                    'pytest',
+                    'devpi-server>=3.0.1',
+                    'devpi-client',
+                    ]
+
+tests_require = ['pytest-cov',
+                 ]
+
+entry_points = {
+    'pytest11': [
+        'devpi_server = pytest_devpi_server',
+    ],
+}
+
+if __name__ == '__main__':
+    kwargs = common_setup('pytest_devpi_server')
+    kwargs.update(dict(
+        name='pytest-devpi-server',
+        description='DevPI server fixture for py.test',
+        author='Edward Easton',
+        author_email='eeaston@gmail.com',
+        classifiers=classifiers,
+        install_requires=install_requires,
+        tests_require=tests_require,
+        packages=find_packages(exclude='tests'),
+        entry_points=entry_points,
+    ))
+    setup(**kwargs)

pytest-devpi-server/tests/integration/test_devpi_server.py

@@ -0,0 +1,31 @@
+import json
+
+
+def test_server(devpi_server):
+    res = devpi_server.api('getjson', '/{}/{}'.format(devpi_server.user, devpi_server.index))
+    assert json.loads(res) == {
+        'result': {
+           'acl_upload': ['testuser'], 
+           'bases': [], 
+           'mirror_whitelist': [], 
+           'projects': [], 
+           'pypi_whitelist': [], 
+           'type': 'stage', 
+           'volatile': True
+        }, 
+        'type': 'indexconfig'
+    }
+
+def test_upload(devpi_server):
+    pkg_dir = devpi_server.workspace / 'pkg'
+    pkg_dir.mkdir_p()
+    setup_py = pkg_dir / 'setup.py'
+    setup_py.write_text("""
+from setuptools import setup
+setup(name='test-foo',
+      version='1.2.3')
+""")
+    pkg_dir.chdir()
+    devpi_server.api('upload')
+    res = devpi_server.api('getjson', '/{}/{}'.format(devpi_server.user, devpi_server.index))
+    assert json.loads(res)['result']['projects'] == ['test-foo']