Add tclk codegen changes (#1023)

* Add initial metadata

* Create nitclk.mak

* Remove numpy restriction

Bug fixed

* Upgrade pip

* Remove specific version of flake8

'\' in comments now need to be escaped

* Add basepython section

* Try using pypy3.5

* Go back to pypy3 and remove basepython block

* Don't need to run build_test for all builds - only 3.6

* Remove warning from coverage

* Undo previous change

* Try disabling faulthandler

* Remove --no-faulthandler

* pypy needs an older version of pytest

See pytest-dev/pytest#5807

* Use platform_python_implementation since implementation_name is not defined on Python2

* Try to get back to the initial coverage

* Do the same thing for build_test

* Remove redundant pip

* Add codecov file to try to control limits for passing

* Enable nitclk build

* There are some files we will need to skip for now

* Add generated nitclk files

* Update generated files

* Can't skip README.rst

* Add new cases for buffers with a converter

* Need to handle ' []' to normalize everything

* Get the session handle name from config instead of hardcoding 'vi'

* Temp commit

* Fix syntax error

* Test new cases

* Force change

* Update generated files

* Update generated files

* Add different target for pr (project) and commit (patch)

* threshold should be number

* Handle no close

* Handle not having a close function

* Update generated files

* Add nifake specific converter for testing

* Add function using test converter

* Add VI parameter

* Change type so I don't need another matcher

* Test using a converter with a buffer

* Update generated files

build/helper/codegen_helper.py

@@ -348,6 +348,7 @@ def _get_ctype_variable_definition_snippet_for_buffers(parameter, parameters, iv
         B600. Output buffer with mechanism passed-in:                              get_ctypes_pointer_for_buffer(value-array.array('d', [0] * buffer_size, library_type=ViInt32)
         B610. Output buffer with mechanism ivi-dance-with-a-twist, QUERY_SIZE:     None
         B620. Output buffer with mechanism ivi-dance-with-a-twist, GET_DATA:       get_ctypes_pointer_for_buffer(value=array.array('d', [0] * buffer_size_ctype.value, library_type=ViInt32)
+        B630. Input buffer of simple types with converter:                         get_ctypes_pointer_for_buffer(value=convert(array.array('d', list)), library_type=visatype.ViReal64)
 
     Return Value (list): each item in the list will be one line needed for the declaration of that parameter
 
@@ -360,6 +361,19 @@ def _get_ctype_variable_definition_snippet_for_buffers(parameter, parameters, iv
 
     if parameter['numpy'] is True and use_numpy_array is True:
         definition = 'get_ctypes_pointer_for_buffer(value={0})  # case B510'.format(parameter['python_name'])
+    elif parameter['direction'] == 'in' and 'python_api_converter_name' in parameter:
+        if custom_type is not None:
+            assert False, 'Custom type {} is not supported with converters'.format(custom_type)
+        else:
+            if parameter['use_array']:
+                # If the incoming type is array.array, we can just use that, otherwise we need to create an array.array that is initialized with the passed in value, which must be iterable
+                array_declaration = '{0}_array = get_ctypes_and_array(value=_converters.{2}({0}), array_type="{1}")  # case B630'.format(parameter['python_name'], get_array_type_for_api_type(parameter['ctypes_type']), parameter['python_api_converter_name'])
+                definitions.append(array_declaration)
+                definition = 'get_ctypes_pointer_for_buffer(value={0}_array, library_type={1}.{2})  # case B630'.format(parameter['python_name'], module_name, parameter['ctypes_type'])
+            elif parameter['use_list']:
+                definition = 'get_ctypes_pointer_for_buffer(value=_converters.{3}({0}), library_type={1}.{2})  # case B630'.format(parameter['python_name'], module_name, parameter['ctypes_type'], parameter['python_api_converter_name'])
+            else:
+                assert False, "Expected either 'use_array' or 'use_list' to be True. Both False."
     elif parameter['direction'] == 'in':
         if custom_type is not None:
             definition = 'get_ctypes_pointer_for_buffer([{0}.{1}(c) for c in {2}], library_type={0}.{1})  # case B540'.format(module_name, parameter['ctypes_type'], parameter['python_name'], parameter['python_name'])
@@ -1139,6 +1153,135 @@ def get_dictionary_snippet(d, indent=4):
         'use_list': True,
         'use_in_python_api': True,
     },
+    {  # 28
+        'ctypes_type': 'ViReal64',
+        'ctypes_type_library_call': 'ctypes.POINTER(ViReal64)',
+        'ctypes_variable_name': 'input_array_2_ctype',
+        'default_value': None,
+        'direction': 'in',
+        'documentation': {'description': 'Input array of floats'},
+        'enum': None,
+        'is_buffer': True,
+        'is_string': False,
+        'use_array': False,
+        'use_list': True,
+        'is_repeated_capability': False,
+        'is_session_handle': False,
+        'library_method_call_snippet': 'input_array_2_ctype',
+        'name': 'inputArray2',
+        'numpy': False,
+        'python_name': 'input_array_2',
+        'python_name_with_default': 'input_array_2=None',
+        'python_name_with_doc_default': 'input_array_2=None',
+        'python_type': 'float',
+        'size': {'mechanism': 'len', 'value': 'inputArraySize2'},
+        'type': 'ViReal64',
+        'use_in_python_api': True,
+    },
+    {  # 29
+        'ctypes_type': 'ViReal64',
+        'ctypes_type_library_call': 'ctypes.POINTER(ViReal64)',
+        'ctypes_variable_name': 'input_array_2_ctype',
+        'default_value': None,
+        'direction': 'in',
+        'documentation': {'description': 'Input array of floats'},
+        'enum': None,
+        'is_buffer': True,
+        'is_string': False,
+        'use_array': False,
+        'use_list': True,
+        'is_repeated_capability': False,
+        'is_session_handle': False,
+        'library_method_call_snippet': 'input_array_2_ctype',
+        'name': 'inputArray2',
+        'numpy': False,
+        'python_api_converter_name': 'convert_to_nitclk_session_num_list',
+        'python_name': 'input_array_2',
+        'python_name_with_default': 'input_array_2=None',
+        'python_name_with_doc_default': 'input_array_2=None',
+        'python_type': 'float',
+        'size': {'mechanism': 'len', 'value': 'inputArraySize2'},
+        'type': 'ViReal64',
+        'use_in_python_api': True,
+    },
+    {  # 30
+        'ctypes_type': 'ViReal64',
+        'ctypes_type_library_call': 'ctypes.POINTER(ViReal64)',
+        'ctypes_variable_name': 'input_array_3_ctype',
+        'default_value': None,
+        'direction': 'in',
+        'documentation': {'description': 'Input array of floats'},
+        'enum': None,
+        'is_buffer': True,
+        'is_string': False,
+        'use_array': True,
+        'use_list': False,
+        'is_repeated_capability': False,
+        'is_session_handle': False,
+        'library_method_call_snippet': 'input_array_3_ctype',
+        'name': 'inputArray3',
+        'numpy': False,
+        'python_api_converter_name': 'convert_to_nitclk_session_num_list',
+        'python_name': 'input_array_3',
+        'python_name_with_default': 'input_array_3=None',
+        'python_name_with_doc_default': 'input_array_3=None',
+        'python_type': 'float',
+        'size': {'mechanism': 'len', 'value': 'inputArraySize3'},
+        'type': 'ViReal64',
+        'use_in_python_api': True,
+    },
+    {  # 31
+        'ctypes_type': 'custom_struct',
+        'ctypes_type_library_call': 'ctypes.POINTER(custom_struct)',
+        'ctypes_variable_name': 'input_array_4_ctype',
+        'default_value': None,
+        'direction': 'in',
+        'documentation': {'description': 'Input array of floats'},
+        'enum': None,
+        'is_buffer': True,
+        'is_string': False,
+        'use_array': False,
+        'use_list': True,
+        'is_repeated_capability': False,
+        'is_session_handle': False,
+        'library_method_call_snippet': 'input_array_3_ctype',
+        'name': 'inputArray4',
+        'numpy': False,
+        'python_api_converter_name': 'convert_to_nitclk_session_num_list',
+        'python_name': 'input_array_4',
+        'python_name_with_default': 'input_array_4=None',
+        'python_name_with_doc_default': 'input_array_4=None',
+        'python_type': 'float',
+        'size': {'mechanism': 'len', 'value': 'inputArraySize4'},
+        'type': 'ViReal64',
+        'use_in_python_api': True,
+    },
+    {  # 32
+        'ctypes_type': 'custom_struct',
+        'ctypes_type_library_call': 'ctypes.POINTER(custom_struct)',
+        'ctypes_variable_name': 'input_array_4_ctype',
+        'default_value': None,
+        'direction': 'in',
+        'documentation': {'description': 'Input array of floats'},
+        'enum': None,
+        'is_buffer': True,
+        'is_string': False,
+        'use_array': False,
+        'use_list': False,
+        'is_repeated_capability': False,
+        'is_session_handle': False,
+        'library_method_call_snippet': 'input_array_3_ctype',
+        'name': 'inputArray4',
+        'numpy': False,
+        'python_api_converter_name': 'convert_to_nitclk_session_num_list',
+        'python_name': 'input_array_4',
+        'python_name_with_default': 'input_array_4=None',
+        'python_name_with_doc_default': 'input_array_4=None',
+        'python_type': 'float',
+        'size': {'mechanism': 'len', 'value': 'inputArraySize4'},
+        'type': 'ViReal64',
+        'use_in_python_api': True,
+    },
 ]
 
 
@@ -1187,6 +1330,11 @@ def test_get_buffer_parameters_for_size_parameter():
     assert params[0] == parameters_for_testing[5]
 
 
+def test_get_ctype_variable_declaration_snippet_case_c010():
+    snippet = get_ctype_variable_declaration_snippet(parameters_for_testing[15], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)
+    assert snippet == ["channel_list_ctype = ctypes.create_string_buffer(self._repeated_capability.encode(self._encoding))  # case C010"]
+
+
 def test_get_ctype_variable_declaration_snippet_case_c050():
     snippet = get_ctype_variable_declaration_snippet(parameters_for_testing[13], parameters_for_testing, IviDanceStep.QUERY_SIZE, config_for_testing, use_numpy_array=False)
     assert snippet == ["a_string_ctype = None  # case C050"]
@@ -1243,8 +1391,8 @@ def test_get_ctype_variable_declaration_snippet_case_s150():
 
 
 def test_get_ctype_variable_declaration_snippet_case_s160():
-    snippet = get_ctype_variable_declaration_snippet(parameters_for_testing[11], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)
-    assert snippet == ["input_array_size_ctype = _visatype.ViInt32(0 if input_array is None else len(input_array))  # case S160"]
+    actual = get_ctype_variable_declaration_snippet(parameters_for_testing[11], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)
+    assert actual == ["input_array_size_ctype = _visatype.ViInt32(0 if input_array is None else len(input_array))  # case S160"]
 
 
 def test_get_ctype_variable_declaration_snippet_case_s170():
@@ -1282,11 +1430,6 @@ def test_get_ctype_variable_declaration_snippet_case_b510():
     assert snippet == ["output_ctype = get_ctypes_pointer_for_buffer(value=output)  # case B510"]
 
 
-def test_get_ctype_variable_declaration_snippet_case_b520():
-    snippet = get_ctype_variable_declaration_snippet(parameters_for_testing[15], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)
-    assert snippet == ["channel_list_ctype = ctypes.create_string_buffer(self._repeated_capability.encode(self._encoding))  # case C010"]
-
-
 def test_get_ctype_variable_declaration_snippet_case_b530():
     snippet = get_ctype_variable_declaration_snippet(parameters_for_testing[16], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)
     assert snippet == ["a_string_ctype = ctypes.create_string_buffer(a_string.encode(self._encoding))  # case C020"]
@@ -1297,7 +1440,7 @@ def test_get_ctype_variable_declaration_snippet_case_b540():
     assert snippet == ["array_in_ctype = get_ctypes_pointer_for_buffer([custom_struct.custom_struct(c) for c in array_in], library_type=custom_struct.custom_struct)  # case B540"]
 
 
-def test_get_ctype_variable_declaration_snippet_case_b550():
+def test_get_ctype_variable_declaration_snippet_case_b550_array():
     actual = get_ctype_variable_declaration_snippet(parameters_for_testing[10], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)
     expected = [
         'input_array_array = get_ctypes_and_array(value=input_array, array_type="d")  # case B550',
@@ -1308,6 +1451,11 @@ def test_get_ctype_variable_declaration_snippet_case_b550():
         assert actual[i] == expected[i]
 
 
+def test_get_ctype_variable_declaration_snippet_case_b550_list():
+    snippet = get_ctype_variable_declaration_snippet(parameters_for_testing[28], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)
+    assert snippet == ["input_array_2_ctype = get_ctypes_pointer_for_buffer(value=input_array_2, library_type=_visatype.ViReal64)  # case B550"]
+
+
 def test_get_ctype_variable_declaration_snippet_case_b560():
     actual = get_ctype_variable_declaration_snippet(parameters_for_testing[3], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)
     expected = [
@@ -1409,6 +1557,38 @@ def test_get_ctype_variable_declaration_snippet_case_b620_list():
         assert actual[i] == expected[i]
 
 
+def test_get_ctype_variable_declaration_snippet_case_b630_array():
+    actual = get_ctype_variable_declaration_snippet(parameters_for_testing[30], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)
+    expected = [
+        'input_array_3_array = get_ctypes_and_array(value=_converters.convert_to_nitclk_session_num_list(input_array_3), array_type="d")  # case B630',
+        'input_array_3_ctype = get_ctypes_pointer_for_buffer(value=input_array_3_array, library_type=_visatype.ViReal64)  # case B630',
+    ]
+    assert len(actual) == len(expected)
+    for i in range(max(len(actual), len(expected))):
+        assert actual[i] == expected[i]
+
+
+def test_get_ctype_variable_declaration_snippet_case_b630_list():
+    snippet = get_ctype_variable_declaration_snippet(parameters_for_testing[29], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)
+    assert snippet == ["input_array_2_ctype = get_ctypes_pointer_for_buffer(value=_converters.convert_to_nitclk_session_num_list(input_array_2), library_type=_visatype.ViReal64)  # case B630"]
+
+
+def test_get_ctype_variable_declaration_snippet_case_b630_custom_type():
+    try:
+        get_ctype_variable_declaration_snippet(parameters_for_testing[31], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)
+        assert False
+    except AssertionError:
+        pass
+
+
+def test_get_ctype_variable_declaration_snippet_case_b630_not_array_or_list():
+    try:
+        get_ctype_variable_declaration_snippet(parameters_for_testing[32], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)
+        assert False
+    except AssertionError:
+        pass
+
+
 def test_get_ctype_variable_declaration_snippet_bad_ivi_dance_step():
     try:
         get_ctype_variable_declaration_snippet(parameters_for_testing[12], parameters_for_testing, IviDanceStep.NOT_APPLICABLE, config_for_testing, use_numpy_array=False)

build/helper/metadata_add_all.py

@@ -292,7 +292,7 @@ def _add_is_session_handle(parameter):
 
 def _fix_type(parameter):
     '''Replace any spaces in the parameter type with an underscore.'''
-    parameter['type'] = parameter['type'].replace('[ ]', '[]').replace(' ', '_')
+    parameter['type'] = parameter['type'].replace('[ ]', '[]').replace(' []', '[]').replace(' ', '_')
 
 
 def _add_use_in_python_api(p, parameters):
@@ -314,7 +314,7 @@ def _setup_init_function(functions, config):
         # Change the init_function information for generating the docstring
         # We are assuming the last parameter is vi out
         for p in init_function['parameters']:
-            if p['name'] == 'vi':
+            if p['name'] == config['session_handle_parameter_name']:
                 p['documentation']['description'] = session_return_text
                 p['type_in_documentation'] = config['module_name'] + '.Session'
                 p['python_name'] = 'session'

build/templates/_converters.py.mako

@@ -176,6 +176,13 @@ def convert_month_to_timedelta(months):
     return datetime.timedelta(days=(30.4167 * months))
 
 
+% if config['module_name'] == 'nifake':
+# nifake specific converter(s) - used only for testing
+def convert_double_each_element(numbers):
+    return [x * 2 for x in numbers]
+
+
+% endif
 # This converter is not called from the normal codegen path for function. Instead it is
 # call from init and is a special case. Also, it just returns a string rather than a ctype object
 def convert_init_with_options_dictionary(values, encoding):

generated/nifake/_converters.py

@@ -172,6 +172,11 @@ def convert_month_to_timedelta(months):
     return datetime.timedelta(days=(30.4167 * months))
 
 
+# nifake specific converter(s) - used only for testing
+def convert_double_each_element(numbers):
+    return [x * 2 for x in numbers]
+
+
 # This converter is not called from the normal codegen path for function. Instead it is
 # call from init and is a special case. Also, it just returns a string rather than a ctype object
 def convert_init_with_options_dictionary(values, encoding):

generated/nifake/_library.py

@@ -22,6 +22,7 @@ def __init__(self, ctypes_library):
         # We cache the cfunc object from the ctypes.CDLL object
         self.niFake_Abort_cfunc = None
         self.niFake_BoolArrayOutputFunction_cfunc = None
+        self.niFake_DoubleAllTheNums_cfunc = None
         self.niFake_EnumArrayOutputFunction_cfunc = None
         self.niFake_EnumInputFunctionWithDefaults_cfunc = None
         self.niFake_FetchWaveform_cfunc = None
@@ -89,6 +90,14 @@ def niFake_BoolArrayOutputFunction(self, vi, number_of_elements, an_array):  # n
                 self.niFake_BoolArrayOutputFunction_cfunc.restype = ViStatus  # noqa: F405
         return self.niFake_BoolArrayOutputFunction_cfunc(vi, number_of_elements, an_array)
 
+    def niFake_DoubleAllTheNums(self, vi, number_count, numbers):  # noqa: N802
+        with self._func_lock:
+            if self.niFake_DoubleAllTheNums_cfunc is None:
+                self.niFake_DoubleAllTheNums_cfunc = self._library.niFake_DoubleAllTheNums
+                self.niFake_DoubleAllTheNums_cfunc.argtypes = [ViSession, ViInt32, ctypes.POINTER(ViReal64)]  # noqa: F405
+                self.niFake_DoubleAllTheNums_cfunc.restype = ViStatus  # noqa: F405
+        return self.niFake_DoubleAllTheNums_cfunc(vi, number_count, numbers)
+
     def niFake_EnumArrayOutputFunction(self, vi, number_of_elements, an_array):  # noqa: N802
         with self._func_lock:
             if self.niFake_EnumArrayOutputFunction_cfunc is None:

generated/nifake/session.py

@@ -782,6 +782,23 @@ def bool_array_output_function(self, number_of_elements):
         errors.handle_error(self, error_code, ignore_warnings=False, is_error_handling=False)
         return [bool(an_array_ctype[i]) for i in range(number_of_elements_ctype.value)]
 
+    @ivi_synchronized
+    def double_all_the_nums(self, numbers):
+        r'''double_all_the_nums
+
+        Test for buffer with converter
+
+        Args:
+            numbers (list of float): numbers is an array of numbers we want to double.
+
+        '''
+        vi_ctype = _visatype.ViSession(self._vi)  # case S110
+        number_count_ctype = _visatype.ViInt32(0 if numbers is None else len(numbers))  # case S160
+        numbers_ctype = get_ctypes_pointer_for_buffer(value=_converters.convert_double_each_element(numbers), library_type=_visatype.ViReal64)  # case B630
+        error_code = self._library.niFake_DoubleAllTheNums(vi_ctype, number_count_ctype, numbers_ctype)
+        errors.handle_error(self, error_code, ignore_warnings=False, is_error_handling=False)
+        return
+
     @ivi_synchronized
     def enum_array_output_function(self, number_of_elements):
         r'''enum_array_output_function