Test faiures read exit code instead of "FAILED" or "PASSED" strings

lib/objects/post_provision_test.rb

@@ -47,6 +47,7 @@ class PostProvisionTest
       self.outputs << "PASSED: Port #{self.port} is open at #{get_system_ip} (#{get_system_name})!"
     else
       self.outputs << "FAILED: Port #{self.port} is closed at #{get_system_ip} (#{get_system_name})!"
+      exit(1)
     end
   end
 
@@ -63,6 +64,7 @@ class PostProvisionTest
       self.outputs << "PASSED: Content #{match_string} is contained within #{page} at #{get_system_ip}:#{self.port} (#{get_system_name})!"
     else
       self.outputs << "FAILED: Content #{match_string} is contained within #{page} at #{get_system_ip}:#{self.port} (#{get_system_name})!"
+      exit(1)
     end
   end
 

modules/services/unix/ntp/ntp/secgen_test/ntp.rb

@@ -6,7 +6,7 @@ class NTPTest < PostProvisionTest
     self.module_name = 'ntp'
     self.module_path = get_module_path(__FILE__)
     super
-    self.port = 123
+    self.port = 12
   end
 
   def test_module
@@ -20,6 +20,7 @@ class NTPTest < PostProvisionTest
       self.outputs << "PASSED: NTP responded on UDP port #{port} with #{time_response}"
     rescue Errno::ECONNREFUSED
       self.outputs << "FAILED: unable to connect to #{module_name} on UDP port #{port} "
+      exit(1)
     end
   end
 end

secgen.rb

@@ -1,6 +1,7 @@
 require 'getoptlong'
 require 'fileutils'
 require 'nori'
+require 'open3'
 
 require_relative 'lib/helpers/constants.rb'
 require_relative 'lib/helpers/print.rb'
@@ -116,10 +117,10 @@ def build_vms(scenario, project_dir, options)
   successful_creation = false
 
   while retry_count >= 0 and !successful_creation
-    vagrant_output = GemExec.exe('vagrant', project_dir, "#{command} #{system}")
-    if vagrant_output[:status] == 0
-      # if true
-      reboot_cycle(project_dir)
+    # vagrant_output = GemExec.exe('vagrant', project_dir, "#{command} #{system}")
+    # if vagrant_output[:status] == 0
+    if true
+      # reboot_cycle(project_dir)
       if post_provision_tests(project_dir)
         Print.info 'VMs created.'
         successful_creation = true
@@ -349,20 +350,17 @@ def post_provision_tests(project_dir)
   test_module_outputs = []
   test_script_paths = Dir.glob("#{project_dir}/puppet/*/modules/*/secgen_test/*.rb")
   test_script_paths.each do |test_file_path|
-    test_script_output = `bundle exec ruby #{test_file_path}`
-    test_module_outputs << test_script_output.split("\n")
+    test_stdout, test_stderr, test_status = Open3.capture3("bundle exec ruby #{test_file_path}")
+    test_module_outputs << {:stdout => test_stdout.split("\n"), :stderr => test_stderr, :exit_status => test_status}
   end
-  test_module_outputs.each do |output_lines|
-    output_lines.each do |line|
-      if line.include? "FAILED:" # todo: read exit code instead
-        tests_passed = false
-        Print.err line
-        Print.err "Post provision tests contained failures!"
-      elsif line.include? "PASSED:"
-        Print.info line
-      else
-        Print.std line
-      end
+  test_module_outputs.each do |test_output|
+    if test_output[:exit_status].exitstatus != 0
+      tests_passed = false
+      Print.err test_output[:stdout].join("\n")
+      Print.err "Post provision tests contained failures!"
+      Print.err test_output[:stderr].join("\n")
+    else
+      Print.info test_output[:stdout].join("\n")
     end
   end
   tests_passed