Add a test case for embedded ruby use and issue #527

Since the problem is a dead lock (inside pthread_cond_wait, on Linux), the test
case has to spawn a separate process which can be killed on failure.

spec/ffi/callback_spec.rb

@@ -775,6 +775,7 @@ module LibTestStdcall
 describe "Callback interop" do
   require 'fiddle'
   require 'fiddle/import'
+  require 'timeout'
 
   module LibTestFFI
     extend FFI::Library
@@ -844,4 +845,22 @@ def assert_callback_in_same_thread_called_once
       LibTestFiddle.testClosureVrV(Fiddle::Pointer[func.to_i])
     end
   end
+
+  # https://github.com/ffi/ffi/issues/527
+  if RUBY_ENGINE == 'ruby' && RUBY_VERSION.split('.').map(&:to_i).pack("C*") >= [2,3,0].pack("C*")
+    it "C outside ffi call stack does not deadlock [#527]" do
+      path = File.join(File.dirname(__FILE__), "embed-test/embed-test.rb")
+      pid = spawn(RbConfig.ruby, "-Ilib", path, { [:out, :err] => "embed-test.log" })
+      begin
+        Timeout.timeout(10){ Process.wait(pid) }
+      rescue Timeout::Error
+        Process.kill(9, pid)
+        raise
+      else
+        if $?.exitstatus != 0
+          raise "external process failed:\n#{ File.read("embed-test.log") }"
+        end
+      end
+    end
+  end
 end

spec/ffi/embed-test/embed-test.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+#
+# This file is part of ruby-ffi.
+# For licensing, see LICENSE.SPECS
+#
+
+# This test specifically avoids calling native code through FFI.
+# Instead, the stock extension mechanism is used. The reason is
+# that the C extension initializes FFI and then calls a callback
+# which deadlocked in earlier FFI versions, see
+# https://github.com/ffi/ffi/issues/527
+
+EXT = File.expand_path("ext/embed_test.so", File.dirname(__FILE__))
+
+old = Dir.pwd
+Dir.chdir(File.dirname(EXT))
+
+nul = File.open("/dev/null")
+make = system('type gmake', { :out => nul, :err => nul }) && 'gmake' || 'make'
+
+# create Makefile
+system(RbConfig.ruby, "extconf.rb")
+
+# compile extension
+unless system(make)
+  raise "Unable to compile \"#{EXT}\""
+end
+
+Dir.chdir(old)
+
+require EXT
+EmbedTest::testfunc

spec/ffi/embed-test/ext/embed.c

@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2017 Thomas Martitz <kugel@rockbox.org>
+ * Copyright (C) 2008-2017, Ruby FFI project contributors
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the Ruby FFI project nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <dlfcn.h>
+
+#include <ruby.h>
+
+const char script[] =
+"require 'rubygems'\n"
+"require 'ffi'\n"
+"module LibWrap\n"
+"  extend FFI::Library\n"
+"  ffi_lib [FFI::CURRENT_PROCESS, '']\n"
+"  callback :completion_function, [:string, :long, :uint8], :void\n"
+"  attach_function :do_work, [:pointer, :completion_function], :int\n"
+"  Callback = Proc.new do |buf_ptr, count, code|\n"
+"    nil\n"
+"  end\n"
+"end\n"
+"\n"
+"LibWrap.do_work(\"test\", LibWrap::Callback)\n";
+
+typedef void completion_function(char *buffer, long count, unsigned char code);
+
+extern int do_work(char *buffer, completion_function *);
+
+static completion_function *ruby_func;
+
+static int success = 0;
+
+int do_work(char *buffer, completion_function *fn)
+{
+  /* Calling fn directly here works */
+  ruby_func = fn;
+  return 0;
+}
+
+static VALUE testfunc(VALUE args);
+
+void Init_embed_test(void)
+{
+  VALUE mod = rb_define_module("EmbedTest");
+  rb_define_module_function(mod, "testfunc", testfunc, 0);
+}
+
+static VALUE testfunc(VALUE self)
+{
+  int state = 0;
+  VALUE ret;
+
+  rb_eval_string_protect(script, &state);
+
+  if (state)
+  {
+    VALUE e = rb_errinfo();
+    ret = rb_funcall(e, rb_intern("message"), 0);
+    fprintf(stderr, "exc %s\n", StringValueCStr(ret));
+    rb_set_errinfo(Qnil);
+    exit(1);
+  }
+  else
+  {
+    /* Calling fn here hangs, because ffi expects an initial ruby stack
+     * frame. Spawn a thread to kill the process, otherwise the deadlock
+     * would prevent completing the test. */
+    ruby_func("hello", 5, 0);
+  }
+}

spec/ffi/embed-test/ext/extconf.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+#
+# This file is part of ruby-ffi.
+# For licensing, see LICENSE.SPECS
+#
+
+if !defined?(RUBY_ENGINE) || RUBY_ENGINE == 'ruby' || RUBY_ENGINE == 'rbx'
+  require "mkmf"
+
+  create_makefile("embed_test")
+end