Skip to content

Non-deterministic test failures in vo.samp #2064

Closed
Closed
Non-deterministic test failures in vo.samp#2064
Labels
Bugsamptesting

Description

@mdboom
mdboom
opened on Feb 7, 2014

@astrofrog: You've probably seen some errors of this type already, but I thought it might be worth documenting them in an issue.

_________________ TestStandardProfileHTTPSHubClient.test_main __________________

self = <astropy.vo.samp.tests.test_standard_profile.TestStandardProfileHTTPSHubClient object at 0xf0969d0>

�[1m    def test_main(self):�[0m
�[1m    �[0m
�[1m        self.client1_id = self.client1.get_public_id()�[0m
�[1m        self.client2_id = self.client2.get_public_id()�[0m
�[1m    �[0m
�[1m        self.metadata1 = {"samp.name": "Client 1",�[0m
�[1m                          "samp.description.text": "Client 1 Description",�[0m
�[1m                          "client.version": "1.1"}�[0m
�[1m    �[0m
�[1m        self.metadata2 = {"samp.name": "Client 2",�[0m
�[1m                          "samp.description.text": "Client 2 Description",�[0m
�[1m                          "client.version": "1.2"}�[0m
�[1m    �[0m
�[1m        # Check that the clients are connected�[0m
�[1m    �[0m
�[1m        assert self.client1.is_connected�[0m
�[1m        assert self.client2.is_connected�[0m
�[1m    �[0m
�[1m        # Check that ping works�[0m
�[1m    �[0m
�[1m        self.client1.ping()�[0m
�[1m        self.client2.ping()�[0m
�[1m    �[0m
�[1m        # Check that get_registered_clients works as expected.�[0m
�[1m    �[0m
�[1m        assert self.client1_id not in self.client1.get_registered_clients()�[0m
�[1m        assert self.client2_id in self.client1.get_registered_clients()�[0m
�[1m        assert self.client1_id in self.client2.get_registered_clients()�[0m
�[1m        assert self.client2_id not in self.client2.get_registered_clients()�[0m
�[1m    �[0m
�[1m        # Check that get_metadata works as expected�[0m
�[1m    �[0m
�[1m        assert self.client1.get_metadata(self.client1_id) == {}�[0m
�[1m        assert self.client1.get_metadata(self.client2_id) == {}�[0m
�[1m        assert self.client2.get_metadata(self.client1_id) == {}�[0m
�[1m        assert self.client2.get_metadata(self.client2_id) == {}�[0m
�[1m    �[0m
�[1m        self.client1.declare_metadata(self.metadata1)�[0m
�[1m    �[0m
�[1m        assert self.client1.get_metadata(self.client1_id) == self.metadata1�[0m
�[1m        assert self.client2.get_metadata(self.client1_id) == self.metadata1�[0m
�[1m        assert self.client1.get_metadata(self.client2_id) == {}�[0m
�[1m        assert self.client2.get_metadata(self.client2_id) == {}�[0m
�[1m    �[0m
�[1m        self.client2.declare_metadata(self.metadata2)�[0m
�[1m    �[0m
�[1m        assert self.client1.get_metadata(self.client1_id) == self.metadata1�[0m
�[1m        assert self.client2.get_metadata(self.client1_id) == self.metadata1�[0m
�[1m        assert self.client1.get_metadata(self.client2_id) == self.metadata2�[0m
�[1m        assert self.client2.get_metadata(self.client2_id) == self.metadata2�[0m
�[1m    �[0m
�[1m        # Check that, without subscriptions, sending a notification from one�[0m
�[1m        # client to another raises an error.�[0m
�[1m    �[0m
�[1m        message = {}�[0m
�[1m        message['samp.mtype'] = "table.load.votable"�[0m
�[1m        message['samp.params'] = {}�[0m
�[1m    �[0m
�[1m        with pytest.raises(SAMPProxyError):�[0m
�[1m            self.client1.notify(self.client2_id, message)�[0m
�[1m    �[0m
�[1m        # Check that there are no currently active subscriptions�[0m
�[1m    �[0m
�[1m        assert self.client1.get_subscribed_clients('table.load.votable') == {}�[0m
�[1m        assert self.client2.get_subscribed_clients('table.load.votable') == {}�[0m
�[1m    �[0m
�[1m        # We now test notifications and calls�[0m
�[1m    �[0m
�[1m        rec1 = Receiver(self.client1)�[0m
�[1m        rec2 = Receiver(self.client2)�[0m
�[1m    �[0m
�[1m        self.client2.bind_receive_notification('table.load.votable',�[0m
�[1m                                               rec2.receive_notification)�[0m
�[1m    �[0m
�[1m        self.client2.bind_receive_call('table.load.votable',�[0m
�[1m                                       rec2.receive_call)�[0m
�[1m    �[0m
�[1m        self.client1.bind_receive_response('test-tag', rec1.receive_response)�[0m
�[1m    �[0m
�[1m        # Check resulting subscriptions�[0m
�[1m    �[0m
�[1m        assert self.client1.get_subscribed_clients('table.load.votable') == {self.client2_id: {}}�[0m
�[1m        assert self.client2.get_subscribed_clients('table.load.votable') == {}�[0m
�[1m    �[0m
�[1m        assert 'table.load.votable' in self.client1.get_subscriptions(self.client2_id)�[0m
�[1m        assert 'table.load.votable' in self.client2.get_subscriptions(self.client2_id)�[0m
�[1m    �[0m
�[1m        # Once we have finished with the calls and notifications, we will�[0m
�[1m        # check the data got across correctly.�[0m
�[1m    �[0m
�[1m        # Test notify�[0m
�[1m    �[0m
�[1m        params = random_params(self.tmpdir)�[0m
�[1m        self.client1.notify(self.client2.get_public_id(),�[0m
�[1m                            {'samp.mtype':'table.load.votable',�[0m
�[1m                             'samp.params':params})�[0m
�[1m    �[0m
�[1m        assert_output('table.load.votable', self.client2.get_private_key(),�[0m
�[1m                      self.client1_id, params, timeout=60)�[0m
�[1m    �[0m
�[1m        params = random_params(self.tmpdir)�[0m
�[1m        self.client1.enotify(self.client2.get_public_id(),�[0m
�[1m                             "table.load.votable", **params)�[0m
�[1m    �[0m
�[1m        assert_output('table.load.votable', self.client2.get_private_key(),�[0m
�[1m                      self.client1_id, params, timeout=60)�[0m
�[1m    �[0m
�[1m        # Test notify_all�[0m
�[1m    �[0m
�[1m        params = random_params(self.tmpdir)�[0m
�[1m        self.client1.notify_all({'samp.mtype':'table.load.votable',�[0m
�[1m                                 'samp.params':params})�[0m
�[1m    �[0m
�[1m        assert_output('table.load.votable', self.client2.get_private_key(),�[0m
�[1m                      self.client1_id, params, timeout=60)�[0m
�[1m    �[0m
�[1m        params = random_params(self.tmpdir)�[0m
�[1m        self.client1.enotify_all("table.load.votable", **params)�[0m
�[1m    �[0m
�[1m        assert_output('table.load.votable', self.client2.get_private_key(),�[0m
�[1m                      self.client1_id, params, timeout=60)�[0m
�[1m    �[0m
�[1m        # Test call�[0m
�[1m    �[0m
�[1m        params = random_params(self.tmpdir)�[0m
�[1m        self.client1.call(self.client2.get_public_id(), 'test-tag',�[0m
�[1m                            {'samp.mtype':'table.load.votable',�[0m
�[1m                             'samp.params':params})�[0m
�[1m    �[0m
�[1m        assert_output('table.load.votable', self.client2.get_private_key(),�[0m
�[1m                      self.client1_id, params, timeout=60)�[0m
�[1m    �[0m
�[1m        params = random_params(self.tmpdir)�[0m
�[1m        self.client1.ecall(self.client2.get_public_id(), 'test-tag',�[0m
�[1m                           "table.load.votable", **params)�[0m
�[1m    �[0m
�[1m        assert_output('table.load.votable', self.client2.get_private_key(),�[0m
�[1m>                     self.client1_id, params, timeout=60)�[0m

astropy/vo/samp/tests/test_standard_profile.py:207: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

mtype = 'table.load.votable'
private_key = '3beab3d6-903e-11e3-a1ca-9fe19912b5cc', sender_id = 'cli#1'
params = {'parameter1': 'abcde', 'parameter2': 1331, 'verification_file': '/tmp/tmpys6o59/PTNlY581naU4djoi'}
timeout = 60

�[1m    def assert_output(mtype, private_key, sender_id, params, timeout=None):�[0m
�[1m        filename = params['verification_file']�[0m
�[1m        start = time.time()�[0m
�[1m        while True:�[0m
�[1m            try:�[0m
�[1m                with open(filename, 'rb') as f:�[0m
�[1m                    rec_mtype = pickle.load(f)�[0m
�[1m                    rec_private_key = pickle.load(f)�[0m
�[1m                    rec_sender_id = pickle.load(f)�[0m
�[1m                    rec_params = pickle.load(f)�[0m
�[1m                break�[0m
�[1m            except (IOError, EOFError):�[0m
�[1m                if timeout is not None and time.time() - start > timeout:�[0m
�[1m>                   raise Exception("Timeout while waiting for file: {0}".format(filename))�[0m
�[1m�[31mE                   Exception: Timeout while waiting for file: /tmp/tmpys6o59/PTNlY581naU4djoi�[0m

astropy/vo/samp/tests/test_helpers.py:36: Exception
------------------------------- Captured stdout --------------------------------
INFO: Hub set for using SSL. [astropy.vo.samp.hub]
INFO: Hub started [astropy.vo.samp.hub]
�[1m�[31m======= 1 failed, 5060 passed, 86 skipped, 10 xfailed in 417.32 seconds ========�[0m

https://travis-ci.org/astropy/astropy/jobs/18432543

Activity

  1. astrofrog
    added
    vo.samp
    on Feb 7, 2014

  2. astrofrog commented on Feb 8, 2014

    @astrofrog
    astrofrog
    on Feb 8, 2014
    Member

    I thought I'd managed to get rid of these... I think I should mark them as xfail until we understand better what the failures are due to. If someone wants to help me understand what the errors are due to, let me know!

  3. astrofrog commented on Mar 16, 2014

    @astrofrog
    astrofrog
    on Mar 16, 2014
    Member

    Has anyone seen these recently? A few came up after I merged #2067, but I think that was mostly from branches that were forked before that merge.

  4. astrofrog
    added this to the v0.4.0 milestone on Mar 16, 2014

  5. cdeil commented on Apr 12, 2014

    @cdeil
    cdeil
    on Apr 12, 2014
    Member

    This is still an issue ... see e.g. here:
    #2276 (comment)
    Apparently when a travis-ci job is re-started the old log is replaced, so the astropy.vo.samp error is no longer visible here: https://travis-ci.org/astropy/astropy/jobs/22730167#L1846

    But here's another astropy.vo.samp timeout test failure in an unrelated PR:
    https://travis-ci.org/astropy/astropy/jobs/22816314#L1384
    Here's a copy:
    https://gist.github.com/cdeil/10524552

  6. cdeil commented on Apr 12, 2014

    @cdeil
    cdeil
    on Apr 12, 2014
    Member

    Another "random" test error from astropy.vo.samp:
    https://travis-ci.org/astropy/astropy/jobs/22779789#L1634
    Copy in case it gets updated after a rebuild:
    https://gist.github.com/cdeil/10525081

    👍 to mark those tests xfail until the issue is fixed.

    And here's a "random" test error from astropy.time:
    https://travis-ci.org/astropy/astropy/jobs/22779790#L4778
    Copy in case it gets updated after a rebuild:
    https://gist.github.com/cdeil/10525174

  7. cdeil
    added 3 commits that reference this issue on Apr 18, 2014

    Mark SAMP server tests xfail (see issue astropy#2064)

    c81868c

    Mark SAMP HTTPS server tests xfail (see issue astropy#2064)

    662c83b

    Mark SAMP HTTPS server tests xfail (see issue astropy#2064)

    6c66e2d
  8. astrofrog
    modified the milestones: v0.4.1, v0.4.0 on May 8, 2014
  9. embray
    modified the milestones: v0.4.2, v0.4.3 on Sep 23, 2014
  10. ktchrn
    added a commit that references this issue on Oct 28, 2014

    Mark SAMP HTTPS server tests xfail (see issue astropy#2064)

    5485a0a
  11. embray
    added
    Bug
    testing
    on Nov 24, 2014
  12. embray
    removed this from the v0.4.3 milestone on Nov 24, 2014

  13. cdeil commented on Aug 11, 2015

    @cdeil
    cdeil
    on Aug 11, 2015
    Member

    Now there's this pytest-ignore-flaky plugin:

    https://mail.python.org/pipermail/pytest-dev/2015-August/003104.html
    https://pypi.python.org/pypi/pytest-ignore-flaky
    https://github.com/schettino72/pytest-ignore-flaky

    Should we use it?
    Do we still have flaky tests in Astropy?

  14. astrofrog commented on Aug 11, 2015

    @astrofrog
    astrofrog
    on Aug 11, 2015
    Member

    Has anyone seen SAMP errors recently? I thought that this now seemed to be fixed? (or maybe we are skipping them - can check later)

  15. mhvk commented on Jan 18, 2017

    @mhvk
    mhvk
    on Jan 18, 2017
    Contributor

    @astrofrog - random failures still seem to occur, though as I restarted the run in #5710, it is hard to find exactly where. From what I recall, though, it seems often related to test_standard_profile and something with HTTPSHub. Now that I look at the file more carefully, there is a note about random failures and using xfail, but not xfail is in fact used. The issues referred in the comment are this one and two (#2126 and #2321) with thoughts about improvements.

    I should add that on my own workstation and laptop (both running Debian), TestStandardProfileHTTPSHub and TestStandardProfileHTTPSHubClient fail reliably. This may be because I'm missing a package or so, but for completeness, here is the output:

    astropy/vo/samp/tests/test_standard_profile.py .EE
================================================================== ERRORS ===================================================================
__________________________________________ ERROR at setup of TestStandardProfileHTTPSHub.test_main __________________________________________

self = <astropy.vo.samp.tests.test_standard_profile.TestStandardProfileHTTPSHub object at 0x7ff7cc1565f8>
method = <bound method TestStandardProfile.test_main of <astropy.vo.samp.tests.test_standard_profile.TestStandardProfileHTTPSHub object at 0x7ff7cc1565f8>>

    def setup_method(self, method):
    
        self.tmpdir = tempfile.mkdtemp()
    
        self.hub = SAMPHubServer(web_profile=False, mode='multiple', pool_size=1,
                                 **self.hub_init_kwargs)
        self.hub.start()
    
        self.client1 = SAMPIntegratedClient(**self.client_init_kwargs)
>       self.client1.connect(hub=self.hub, pool_size=1, **self.client_connect_kwargs)

astropy/vo/samp/tests/test_standard_profile.py:51: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
astropy/vo/samp/integrated_client.py:184: in connect
    cert_reqs, ca_certs, ssl_version, pool_size)
astropy/vo/samp/hub_proxy.py:128: in connect
    self.ping()
astropy/vo/samp/hub_proxy.py:165: in ping
    return self._samp_hub.ping()
astropy/vo/samp/utils.py:65: in __call__
    response = function(*args, **kwrds)
/usr/lib/python3.5/xmlrpc/client.py:1092: in __call__
    return self.__send(self.__name, args)
/usr/lib/python3.5/xmlrpc/client.py:1432: in __request
    verbose=self.__verbose
/usr/lib/python3.5/xmlrpc/client.py:1134: in request
    return self.single_request(host, handler, request_body, verbose)
/usr/lib/python3.5/xmlrpc/client.py:1146: in single_request
    http_conn = self.send_request(host, handler, request_body, verbose)
/usr/lib/python3.5/xmlrpc/client.py:1259: in send_request
    self.send_content(connection, request_body)
/usr/lib/python3.5/xmlrpc/client.py:1289: in send_content
    connection.endheaders(request_body)
/usr/lib/python3.5/http/client.py:1103: in endheaders
    self._send_output(message_body)
/usr/lib/python3.5/http/client.py:934: in _send_output
    self.send(msg)
/usr/lib/python3.5/http/client.py:877: in send
    self.connect()
astropy/vo/samp/ssl_utils.py:62: in connect
    sslconn = ssl.wrap_socket(sock, **kwargs)
/usr/lib/python3.5/ssl.py:1077: in wrap_socket
    ciphers=ciphers)
/usr/lib/python3.5/ssl.py:760: in __init__
    self.do_handshake()
/usr/lib/python3.5/ssl.py:996: in do_handshake
    self._sslobj.do_handshake()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <ssl.SSLObject object at 0x7ff7cc1ab358>

    def do_handshake(self):
        """Start the SSL/TLS handshake."""
>       self._sslobj.do_handshake()
E       ConnectionResetError: [Errno 104] Connection reset by peer

/usr/lib/python3.5/ssl.py:641: ConnectionResetError
----------------------------------------------------------- Captured stdout setup -----------------------------------------------------------
INFO: Hub set for using SSL. [astropy.vo.samp.hub]
INFO: Hub started [astropy.vo.samp.hub]
_______________________________________ ERROR at setup of TestStandardProfileHTTPSHubClient.test_main _______________________________________

self = <astropy.vo.samp.tests.test_standard_profile.TestStandardProfileHTTPSHubClient object at 0x7ff7c9bd1588>
method = <bound method TestStandardProfile.test_main of <astropy.vo.samp.tests.test_standard_profile.TestStandardProfileHTTPSHubClient object at 0x7ff7c9bd1588>>

    def setup_method(self, method):
    
        self.tmpdir = tempfile.mkdtemp()
    
        self.hub = SAMPHubServer(web_profile=False, mode='multiple', pool_size=1,
                                 **self.hub_init_kwargs)
        self.hub.start()
    
        self.client1 = SAMPIntegratedClient(**self.client_init_kwargs)
>       self.client1.connect(hub=self.hub, pool_size=1, **self.client_connect_kwargs)

astropy/vo/samp/tests/test_standard_profile.py:51: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
astropy/vo/samp/integrated_client.py:184: in connect
    cert_reqs, ca_certs, ssl_version, pool_size)
astropy/vo/samp/hub_proxy.py:128: in connect
    self.ping()
astropy/vo/samp/hub_proxy.py:165: in ping
    return self._samp_hub.ping()
astropy/vo/samp/utils.py:65: in __call__
    response = function(*args, **kwrds)
/usr/lib/python3.5/xmlrpc/client.py:1092: in __call__
    return self.__send(self.__name, args)
/usr/lib/python3.5/xmlrpc/client.py:1432: in __request
    verbose=self.__verbose
/usr/lib/python3.5/xmlrpc/client.py:1134: in request
    return self.single_request(host, handler, request_body, verbose)
/usr/lib/python3.5/xmlrpc/client.py:1146: in single_request
    http_conn = self.send_request(host, handler, request_body, verbose)
/usr/lib/python3.5/xmlrpc/client.py:1259: in send_request
    self.send_content(connection, request_body)
/usr/lib/python3.5/xmlrpc/client.py:1289: in send_content
    connection.endheaders(request_body)
/usr/lib/python3.5/http/client.py:1103: in endheaders
    self._send_output(message_body)
/usr/lib/python3.5/http/client.py:934: in _send_output
    self.send(msg)
/usr/lib/python3.5/http/client.py:877: in send
    self.connect()
astropy/vo/samp/ssl_utils.py:62: in connect
    sslconn = ssl.wrap_socket(sock, **kwargs)
/usr/lib/python3.5/ssl.py:1077: in wrap_socket
    ciphers=ciphers)
/usr/lib/python3.5/ssl.py:760: in __init__
    self.do_handshake()
/usr/lib/python3.5/ssl.py:996: in do_handshake
    self._sslobj.do_handshake()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <ssl.SSLObject object at 0x7ff7c9bd1c18>

    def do_handshake(self):
        """Start the SSL/TLS handshake."""
>       self._sslobj.do_handshake()
E       ConnectionResetError: [Errno 104] Connection reset by peer

/usr/lib/python3.5/ssl.py:641: ConnectionResetError
----------------------------------------------------------- Captured stdout setup -----------------------------------------------------------
INFO: Hub set for using SSL. [astropy.vo.samp.hub]
INFO: Hub started [astropy.vo.samp.hub]
======================================== 50 passed, 19 skipped, 1 xfailed, 2 error in 10.90 seconds =========================================
  16. astrofrog
    closed this as completedin #6201on Jun 15, 2017
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    Bugsamptesting

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions