Python 3.6 broken on LX

[Smithx10]

Confirmed this works in Bhyve, but not in LX-CentOS, or LX-Ubuntu-16.04

yum install https://centos7.iuscommunity.org/ius-release.rpm -y 
yum install python36u python36u-devel python36u-pip -y 
python3.6 -m pip install --upgrade pip
python3.6 -m pip install jupyter

error:

[root@17fbf32d-b068-6321-934b-8a137a263382 ~]# jupyter notebook
Traceback (most recent call last):
  File "/usr/bin/jupyter-notebook", line 6, in <module>
    from notebook.notebookapp import main
  File "/usr/lib/python3.6/site-packages/notebook/notebookapp.py", line 79, in <module>
    from .base.handlers import Template404, RedirectWithParams
  File "/usr/lib/python3.6/site-packages/notebook/base/handlers.py", line 32, in <module>
    import prometheus_client
  File "/usr/lib/python3.6/site-packages/prometheus_client/__init__.py", line 7, in <module>
    from . import process_collector
  File "/usr/lib/python3.6/site-packages/prometheus_client/process_collector.py", line 96, in <module>
    PROCESS_COLLECTOR = ProcessCollector()
  File "/usr/lib/python3.6/site-packages/prometheus_client/process_collector.py", line 42, in __init__
    registry.register(self)
  File "/usr/lib/python3.6/site-packages/prometheus_client/registry.py", line 24, in register
    names = self._get_names(collector)
  File "/usr/lib/python3.6/site-packages/prometheus_client/registry.py", line 64, in _get_names
    for metric in desc_func():
  File "/usr/lib/python3.6/site-packages/prometheus_client/process_collector.py", line 84, in collect
    value=float(line.split()[3]))
ValueError: could not convert string to float: b'unlimited'

[Smithx10]

File "/usr/lib/python3.6/site-packages/prometheus_client/process_collector.py", line 84, in collect
value=float(line.split()[3])) couldn't handle if unlimited was set... im stupid. Putting a bug in there.

[Smithx10]

Seem's that the behavior is different in LX than regular CentOS.

The following is printing unlimited for ulimit -Sn, but ulimit shows an int.

[root@85d23256-22ff-4839-fdfb-8cf32fabcd81 ~]# ulimit -Sn
65536
[root@85d23256-22ff-4839-fdfb-8cf32fabcd81 ~]# ulimit -Hn
65536

class ProcessCollector(object):
    """Collector for Standard Exports such as cpu and memory."""

    def __init__(self, namespace='', pid=lambda: 'self', proc='/proc', registry=REGISTRY):
        self._namespace = namespace
        self._pid = pid
        self._proc = proc
        if namespace:
            self._prefix = namespace + '_process_'
        else:
            self._prefix = 'process_'
        self._ticks = 100.0
        try:
            self._ticks = os.sysconf('SC_CLK_TCK')
        except (ValueError, TypeError, AttributeError):
            pass

        # This is used to test if we can access /proc.
        self._btime = 0
        try:
            self._btime = self._boot_time()
        except IOError:
            pass
        if registry:
            registry.register(self)

    def _boot_time(self):
        with open(os.path.join(self._proc, 'stat'), 'rb') as stat:
            for line in stat:
                if line.startswith(b'btime '):
                    return float(line.split()[1])

    def collect(self):
        if not self._btime:
            return []

        pid = os.path.join(self._proc, str(self._pid()).strip())

        result = []
        try:
            with open(os.path.join(pid, 'stat'), 'rb') as stat:
                parts = (stat.read().split(b')')[-1].split())

            vmem = GaugeMetricFamily(self._prefix + 'virtual_memory_bytes',
                                          'Virtual memory size in bytes.', value=float(parts[20]))
            rss = GaugeMetricFamily(self._prefix + 'resident_memory_bytes', 'Resident memory size in bytes.',
                                         value=float(parts[21]) * _PAGESIZE)
            start_time_secs = float(parts[19]) / self._ticks
            start_time = GaugeMetricFamily(self._prefix + 'start_time_seconds',
                                                'Start time of the process since unix epoch in seconds.',
                                                value=start_time_secs + self._btime)
            utime = float(parts[11]) / self._ticks
            stime = float(parts[12]) / self._ticks
            cpu = CounterMetricFamily(self._prefix + 'cpu_seconds_total',
                                           'Total user and system CPU time spent in seconds.',
                                           value=utime + stime)
            result.extend([vmem, rss, start_time, cpu])
        except IOError:
            pass

        try:
            with open(os.path.join(pid, 'limits'), 'rb') as limits:
                for line in limits:
                    print(line)
                    if line.startswith(b'Max open file'):
                        max_fds = GaugeMetricFamily(self._prefix + 'max_fds',
                                                         'Maximum number of open file descriptors.',
                                                         value=float(line.split()[3]))
                        break

Python Printing unlimited in Soft Limit

[root@85d23256-22ff-4839-fdfb-8cf32fabcd81 ~]# jupyter notebook
b'Limit                     Soft Limit           Hard Limit           Units     \n'
b'Max cpu time              unlimited            unlimited            seconds   \n'
b'Max file size             unlimited            9223372036854775807  bytes     \n'
b'Max data size             unlimited            unlimited            bytes     \n'
b'Max stack size            10485760             35184372088832       bytes     \n'
b'Max core file size        unlimited            9223372036854775807  bytes     \n'
b'Max resident set          unlimited            2147483648           bytes     \n'
b'Max processes             unlimited            4000                 processes \n'
b'Max open files            unlimited            65536                files     \n'

[Smithx10]

Looks like its just reading from /proc/self/limits and perhaps getting different information?

[root@85d23256-22ff-4839-fdfb-8cf32fabcd81 ~]# cat /proc/self/limits
Limit                     Soft Limit           Hard Limit           Units
Max cpu time              unlimited            unlimited            seconds
Max file size             unlimited            9223372036854775807  bytes
Max data size             unlimited            unlimited            bytes
Max stack size            10485760             35184372088832       bytes
Max core file size        unlimited            9223372036854775807  bytes
Max resident set          unlimited            2147483648           bytes
Max processes             unlimited            4000                 processes
Max open files            unlimited            65536                files
Max locked memory         unlimited            2147483648           bytes
Max address space         unlimited            unlimited            bytes
Max file locks            unlimited            unlimited            locks
Max pending signals       128                  512                  signals
Max msgqueue size         unlimited            8192                 bytes

[Smithx10]

I think this demonstrates the issue:
LX

arch@archlinux /g/rfd ❯❯❯ tt ssh demo1                                                                                                                                                                                                                                                                                                                       ✘ 130 master ✭
Warning: Permanently added '10.45.136.154' (ECDSA) to the list of known hosts.
Last login: Sat Mar 16 17:54:29 2019 from smith-arch.inst.cloudops-dev.us-east-1.bdf-cloud.iqvia.net
   __        .                   .
 _|  |_      | .-. .  . .-. :--. |-
|_    _|     ;|   ||  |(.-' |  | |
  |__|   `--'  `-' `;-| `-' '  ' `-'
                   /  ;  Instance (CentOS 7.4 20180323)
                   `-'   https://docs.joyent.com/images/container-native-linux

[root@85d23256-22ff-4839-fdfb-8cf32fabcd81 ~]# cat /proc/self/limits
Limit                     Soft Limit           Hard Limit           Units
Max cpu time              unlimited            unlimited            seconds
Max file size             unlimited            9223372036854775807  bytes
Max data size             unlimited            unlimited            bytes
Max stack size            10485760             35184372088832       bytes
Max core file size        unlimited            9223372036854775807  bytes
Max resident set          unlimited            2147483648           bytes
Max processes             unlimited            4000                 processes
Max open files            unlimited            65536                files
Max locked memory         unlimited            2147483648           bytes
Max address space         unlimited            unlimited            bytes
Max file locks            unlimited            unlimited            locks
Max pending signals       128                  512                  signals
Max msgqueue size         unlimited            8192                 bytes
[root@85d23256-22ff-4839-fdfb-8cf32fabcd81 ~]# ulimit -Sn
65536
[root@85d23256-22ff-4839-fdfb-8cf32fabcd81 ~]# ulimit -Hn
65536

Linux

arch@archlinux ~ ❯❯❯ tt ssh demo
Warning: Permanently added '10.45.136.94' (ECDSA) to the list of known hosts.

   __        .                   .
 _|  |_      | .-. .  . .-. :--. |-
|_    _|     ;|   ||  |(.-' |  | |
  |__|   `--'  `-' `;-| `-' '  ' `-'
                   /  ;  Instance (CentOS 7.6 20190122)
                   `-'   https://docs.joyent.com/images/linux/centos

[root@demo ~]#
[root@demo ~]#
[root@demo ~]# cat /proc/self/limits
Limit                     Soft Limit           Hard Limit           Units
Max cpu time              unlimited            unlimited            seconds
Max file size             unlimited            unlimited            bytes
Max data size             unlimited            unlimited            bytes
Max stack size            8388608              unlimited            bytes
Max core file size        0                    unlimited            bytes
Max resident set          unlimited            unlimited            bytes
Max processes             7179                 7179                 processes
Max open files            1024                 4096                 files
Max locked memory         65536                65536                bytes
Max address space         unlimited            unlimited            bytes
Max file locks            unlimited            unlimited            locks
Max pending signals       7179                 7179                 signals
Max msgqueue size         819200               819200               bytes
Max nice priority         0                    0
Max realtime priority     0                    0
Max realtime timeout      unlimited            unlimited            us
[root@demo ~]# ulimit -Sn
1024
[root@demo ~]# ulimit -Hn
4096