Standardise record value setting/getting

.github/workflows/code.yml

@@ -2,7 +2,15 @@ name: Code CI
 
 on:
   push:
+    branches:
+      - master
+      - main
+    tags:
+      - "*"
   pull_request:
+  schedule:
+    # Run every Monday at 8am
+    - cron: '0 8 * * MON'
 
 jobs:
   lint:
@@ -30,15 +38,18 @@ jobs:
         python: [cp36, cp37, cp38, cp39]
 
         include:
-          # Put coverage in the project directory for mac
+          # Put coverage and results files in the project directory for mac
           - os: macos-latest
             cov_file: "{project}/dist/coverage.xml"
+            results_file: "{project}/dist/pytest-results.xml"
           # And for windows
           - os: windows-latest
             cov_file: "{project}/dist/coverage.xml"
-          # But put coverage in the output dir mounted in docker for linux
+            results_file: "{project}/dist/pytest-results.xml"
+          # But put coverage and results files in the output dir mounted in docker for linux
           - os: ubuntu-latest
             cov_file: /output/coverage.xml
+            results_file: /output/pytest-results.xml
           # Build an sdist on linux so it has the right line endings
           - os: ubuntu-latest
             sdist: true
@@ -60,7 +71,8 @@ jobs:
           python-version: "3.7"
 
       - name: Install Python Dependencies
-        run: pip install build cibuildwheel==1.* # The 2.* releases dropped Python2 support.
+        # Pin cibuildwheel due to https://github.com/pypa/cibuildwheel/issues/962
+        run: pip install build cibuildwheel>=2.3.1
 
       - name: Build Sdist
         if: matrix.sdist
@@ -69,14 +81,17 @@ jobs:
       - name: Build Wheel
         run: cibuildwheel --output-dir dist
         env:
+          # Force old behaviour of pip - see https://github.com/pypa/cibuildwheel/issues/962
+          CIBW_ENVIRONMENT: PIP_USE_DEPRECATED=out-of-tree-build
           CIBW_BUILD: ${{ matrix.python }}*64
           CIBW_TEST_EXTRAS: dev
-          CIBW_TEST_COMMAND: pytest {project}/tests --cov-report xml:${{ matrix.cov_file }}
+          CIBW_TEST_COMMAND: pytest {project}/tests --cov-report xml:${{ matrix.cov_file }} --junit-xml=${{ matrix.results_file }}
           # Disable auditwheel as it isn't compatible with setuptools_dso approach
           # https://github.com/mdavidsaver/setuptools_dso/issues/17
           CIBW_REPAIR_WHEEL_COMMAND: ""
-          CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
-          CIBW_ENVIRONMENT_LINUX: SETUPTOOLS_DSO_PLAT_NAME=manylinux1_x86_64
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_ENVIRONMENT_LINUX: SETUPTOOLS_DSO_PLAT_NAME=manylinux2014_x86_64
+          CIBW_SKIP: "*-musllinux*" # epicscorelibs doesn't build on musllinux platforms
 
       - name: Upload Wheel and Sdist
         uses: actions/upload-artifact@v2
@@ -90,6 +105,30 @@ jobs:
           name: ${{ matrix.os }}/${{ matrix.python }}
           directory: dist
 
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: Unit Test Results (${{ matrix.os }}-${{ matrix.python }})
+          path: dist/pytest-results.xml
+
+  publish-test-results:
+    name: Publish Unit Tests Results
+    needs: build
+    runs-on: ubuntu-latest
+    if: always()
+
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v2
+        with:
+          path: artifacts
+
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        with:
+          files: artifacts/**/*.xml
+
   release:
     needs: [build]
     runs-on: ubuntu-latest
@@ -115,4 +154,4 @@ jobs:
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.pypi_token }}
-        run: pipx run twine upload dist/*
+        run: pipx run twine upload dist/*

.github/workflows/docs.yml

@@ -2,6 +2,12 @@ name: Docs CI
 
 on:
   push:
+    branches:
+      # Add more branches here to publish docs from other branches
+      - master
+      - main
+    tags:
+      - "*"
   pull_request:
 
 jobs:

CHANGELOG.rst

@@ -13,6 +13,10 @@ Changed:
 
 - `Remove python2 support <../../pull/64>`
 - `Default DISP to TRUE for all In records <../../pull/74>`
+- `Overhaul record value setting/getting <../../pull/60>`
+
+Added:
+- New longStringIn/longStringOut record types
 
 3.2.1_ - 2021-11-25
 -------------------

docs/reference/api.rst

@@ -267,7 +267,7 @@ All functions return a wrapped `ProcessDeviceSupportIn` or
         mbbOut(name, *options, **fields)
 
     Create ``mbbi`` and ``mbbo`` records.  Up to 16 options can be specified as
-    either an option name or a tuple of tw fields.  The name or first field of
+    either an option name or a tuple of two fields.  The name or first field of
     the tuple names the option, and the second optional field is the option
     severity.  For example::
 
@@ -308,13 +308,28 @@ All functions return a wrapped `ProcessDeviceSupportIn` or
     if given, or defaults to ``'FLOAT'``.
 
 
-The following function generates a specialised record.
+The following functions generates specialised records.
 
 ..  function:: Action(name, **fields)
 
     Creates a record (using `boolOut`) which will always call the `on_update`
-    method when processed.  Used for action records.  The `on_update` keyword
-    should always be passed.
+    method when processed.  Used for action records.  Either the `on_update` or
+    `on_update_name` keyword should always be passed.
+
+..  function::
+        longStringIn(name, **fields)
+        longStringOut(name, **fields)
+
+    Creates a Waveform record specialised to contain Python (Unicode) strings.
+    Only accepts string values, and only returns string values. `initial_value`
+    and ``length`` keywords accepted as per Waveform records.
+
+    If no ``length`` or `initial_value` is provided a default length of 256 is used.
+
+    .. note::
+        If you wish to store byte strings, use a ``WaveformIn/Out`` record
+
+
 
 
 The following functions manage record names.  The record device name must be
@@ -328,8 +343,7 @@ record creation function.
     Sets up the prefix part of the record name, referred to here as the "device"
     part.  This function must be called before creating any records.
     Note that
-    only this function need be used, the three other functions below are
-    entirely optional.
+    only this function need be used, function below is entirely optional.
 
 ..  function:: UnsetDevice()
 

pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
-requires = ["setuptools", "wheel", "setuptools_dso>=2.1", "epicscorelibs>=7.0.6.99.1.0"]
+requires = ["setuptools", "wheel", "setuptools_dso>=2.1", "epicscorelibs==7.0.6.99.1.0"]
 build-backend = "setuptools.build_meta:__legacy__"

softioc/__init__.py

@@ -18,10 +18,10 @@
 # Need to do this before calling anything in device.py
 iocshRegisterCommon()
 for dbd in ('base.dbd', 'PVAServerRegister.dbd', 'qsrv.dbd'):
-    dbLoadDatabase(dbd, os.path.join(path.base_path, "dbd"), None)
-dbLoadDatabase("devIocStats.dbd", os.path.dirname(__file__), None)
+    dbLoadDatabase(dbd, os.path.join(path.base_path, 'dbd'), None)
+dbLoadDatabase('devIocStats.dbd', os.path.dirname(__file__), None)
 
 if registerRecordDeviceDriver(pdbbase):
     raise RuntimeError('Error registering')
 
-__all__ = ["__version__"]
+__all__ = ['__version__']

softioc/builder.py

@@ -7,7 +7,8 @@
 InitialiseDbd()
 LoadDbdFile(os.path.join(os.path.dirname(__file__), 'device.dbd'))
 
-from . import pythonSoftIoc  # noqa
+from . import device, pythonSoftIoc  # noqa
+
 PythonDevice = pythonSoftIoc.PythonDevice()
 
 
@@ -64,7 +65,7 @@ def longOut(name, DRVL=None, DRVH=None, EGU=None, **fields):
     'EI', 'NI', 'TE', 'EL', 'TV', 'TT', 'FT', 'FF']     # 8-15
 
 # All the severity strings supported by <prefix>SV
-_severityStrings = ["NO_ALARM", "MINOR", "MAJOR", "INVALID"]
+_severityStrings = ['NO_ALARM', 'MINOR', 'MAJOR', 'INVALID']
 
 # Converts a list of (option [,severity]) values or tuples into field settings
 # suitable for mbbi and mbbo records.
@@ -78,7 +79,7 @@ def process_value(prefix, value, option, severity=None):
                 severity = _severityStrings[severity]
             fields[prefix + 'SV'] = severity
     # zip() silently ignores extra values in options, so explicitly check length
-    assert len(options) <= 16, "May not specify more than 16 enum values"
+    assert len(options) <= 16, 'May not specify more than 16 enum values'
     for prefix, (value, option) in zip(_mbbPrefixes, enumerate(options)):
         if isinstance(option, tuple):
             # The option is tuple consisting of the option name and an optional
@@ -108,31 +109,28 @@ def Action(name, **fields):
     return boolOut(name, always_update = True, **fields)
 
 
-# Converts numpy character code to FTVL value.
-NumpyCharCodeToFtvl = {
-    # The following type codes are supported directly:
-    'B':    'UCHAR',        # ubyte
-    'b':    'CHAR',         # byte
-    'H':    'USHORT',       # ushort
-    'h':    'SHORT',        # short
-    'i':    'LONG',         # intc
-    'L':    'ULONG',        # uint
-    'l':    'LONG',         # int_
-    'f':    'FLOAT',        # single
-    'd':    'DOUBLE',       # float_
-    'S':    'STRING',       # str_
-
-    # The following type codes are weakly supported by pretending that
-    # they're related types.
-    '?':    'CHAR',         # bool_
-    'p':    'LONG',         # intp
-    'I':    'ULONG',        # uintc
-    'P':    'ULONG',        # uintp
-
-    # The following type codes are not supported at all:
-    #   q   longlong        Q   ulonglong       g   longfloat
-    #   F   csingle         D   complex_        G   clongfloat
-    #   O   object_         U   unicode_        V   void
+# Converts numpy dtype name to FTVL value.
+NumpyDtypeToDbf = {
+    'int8':     'CHAR',
+    'uint8':    'UCHAR',
+    'int16':    'SHORT',
+    'uint16':   'USHORT',
+    'int32':    'LONG',
+    'uint32':   'ULONG',
+    'float32':  'FLOAT',
+    'float64':  'DOUBLE',
+}
+
+# Coverts FTVL string to numpy type
+DbfStringToNumpy = {
+    'CHAR':     'int8',
+    'UCHAR':    'uint8',
+    'SHORT':    'int16',
+    'USHORT':   'uint16',
+    'LONG':     'int32',
+    'ULONG':    'uint32',
+    'FLOAT':    'float32',
+    'DOUBLE':   'float64',
 }
 
 
@@ -146,29 +144,49 @@ def _waveform(value, fields):
         assert not value, 'Can\'t specify initial value twice!'
         value = (fields.pop('initial_value'),)
 
+    # Datatype can be specified as keyword argument, taken from FTVL, or derived
+    # from the initial value
+    if 'datatype' in fields:
+        assert 'FTVL' not in fields, \
+            'Can\'t specify FTVL and datatype together'
+        datatype = fields.pop('datatype')
+        if datatype == int or datatype == 'int':
+            # Convert Python int to 32-bit integer, it's all we can handle
+            datatype = numpy.dtype('int32')
+        else:
+            datatype = numpy.dtype(datatype)
+    elif 'FTVL' in fields:
+        datatype = numpy.dtype(DbfStringToNumpy[fields['FTVL']])
+    else:
+        # No datatype specified, will have to infer from initial value
+        datatype = None
+
     if value:
-        # If a value is specified it should be the *only* non keyword
-        # argument.
+        # If a value is specified it should be the *only* non keyword argument.
         value, = value
-        value = numpy.array(value)
-        fields['initial_value'] = value
-
-        # Pick up default length and datatype from initial value
-        length = len(value)
-        FTVL = NumpyCharCodeToFtvl[value.dtype.char]
+        initial_value = device._require_waveform(value, datatype)
+        length = fields.pop('length', len(initial_value))
+
+        # Special case for [u]int64: if the initial value comes in as 64 bit
+        # integers we cannot represent that, so recast it as [u]int32
+        if datatype is None:
+            if initial_value.dtype == numpy.int64:
+                initial_value = numpy.require(initial_value, numpy.int32)
+            elif initial_value.dtype == numpy.uint64:
+                initial_value = numpy.require(initial_value, numpy.uint32)
     else:
-        # No value specified, so require length and datatype to be specified.
+        initial_value = numpy.array([], dtype = datatype)
         length = fields.pop('length')
-        FTVL = 'FLOAT'
+    datatype = initial_value.dtype
 
-    datatype = fields.pop('datatype', None)
-    if datatype is not None:
-        assert 'FTVL' not in fields, \
-            'Can\'t specify FTVL and datatype together'
-        FTVL = NumpyCharCodeToFtvl[numpy.dtype(datatype).char]
+    assert length > 0, 'Array cannot be of zero length'
+
+    fields['initial_value'] = initial_value
+    fields['_wf_nelm'] = length
+    fields['_wf_dtype'] = datatype
 
     fields['NELM'] = length
-    fields.setdefault('FTVL', FTVL)
+    fields['FTVL'] = NumpyDtypeToDbf[datatype.name]
 
 
 def Waveform(name, *value, **fields):
@@ -182,6 +200,32 @@ def WaveformOut(name, *value, **fields):
     return PythonDevice.waveform_out(name, **fields)
 
 
+def _long_string(fields):
+    if 'length' in fields:
+        length = fields.pop('length')
+    elif 'initial_value' in fields:
+        length = len(fields['initial_value'].encode(errors = 'replace')) + 1
+    else:
+        # Default length of 256
+        length = 256
+
+    fields.setdefault('initial_value', '')
+    fields['_wf_nelm'] = length
+    fields['_wf_dtype'] = numpy.dtype('uint8')
+
+    fields['NELM'] = length
+    fields['FTVL'] = 'UCHAR'
+
+
+def longStringIn(name, **fields):
+    _long_string(fields)
+    return _in_record('long_stringin', name, **fields)
+
+def longStringOut(name, **fields):
+    _long_string(fields)
+    return PythonDevice.long_stringout(name, **fields)
+
+
 
 # ----------------------------------------------------------------------------
 #  Support routines for builder
@@ -227,6 +271,7 @@ def UnsetDevice():
     'stringIn', 'stringOut',
     'mbbIn',    'mbbOut',
     'Waveform', 'WaveformOut',
+    'longStringIn', 'longStringOut',
     'Action',
     # Other builder support functions
     'LoadDatabase',

softioc/device.dbd

@@ -10,5 +10,7 @@ device(waveform,  INST_IO, devPython_waveform,  "Python")
 device(mbbi,      INST_IO, devPython_mbbi,      "Python")
 device(mbbo,      INST_IO, devPython_mbbo,      "Python")
 
-# Waveforms as "out" devices need special treatment.
+# A variety of specialised versions of waveform need special treatment.
 device(waveform,  INST_IO, devPython_waveform_out, "PythonWfOut")
+device(waveform,  INST_IO, devPython_long_stringin, "PythonLongStringIn")
+device(waveform,  INST_IO, devPython_long_stringout, "PythonLongStringOut")