Dali 3D User Interface Engine
test-harness.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2014 Samsung Electronics Co., Ltd.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "test-harness.h"
18 #include <stdlib.h>
19 #include <sys/types.h>
20 #include <sys/wait.h>
21 #include <unistd.h>
22 #include <vector>
23 #include <map>
24 #include <cstring>
25 #include <testcase.h>
26 
27 namespace TestHarness
28 {
29 
30 typedef std::map<int, TestCase> RunningTestCases;
31 
32 const char* basename(const char* path)
33 {
34  const char* ptr=path;
35  const char* slash=NULL;
36  for( ; *ptr != '\0' ; ++ptr )
37  {
38  if(*ptr == '/') slash=ptr;
39  }
40  if(slash != NULL) ++slash;
41  return slash;
42 }
43 
44 int RunTestCase( struct ::testcase_s& testCase )
45 {
46  int result = EXIT_STATUS_TESTCASE_FAILED;
47 
48 // dont want to catch exception as we want to be able to get
49 // gdb stack trace from the first error
50 // by default tests should all always pass with no exceptions
51  if( testCase.startup )
52  {
53  testCase.startup();
54  }
55  result = testCase.function();
56  if( testCase.cleanup )
57  {
58  testCase.cleanup();
59  }
60 
61  return result;
62 }
63 
64 
65 int RunTestCaseInChildProcess( struct ::testcase_s& testCase, bool suppressOutput )
66 {
67  int testResult = EXIT_STATUS_TESTCASE_FAILED;
68 
69  int pid = fork();
70  if( pid == 0 ) // Child process
71  {
72  if( suppressOutput )
73  {
74  close(STDOUT_FILENO);
75  close(STDERR_FILENO);
76  }
77  else
78  {
79  printf("\n");
80  for(int i=0; i<80; ++i) printf("#");
81  printf("\nTC: %s\n", testCase.name);
82  fflush(stdout);
83  }
84 
85  int status = RunTestCase( testCase );
86 
87  if( ! suppressOutput )
88  {
89  fflush(stdout);
90  fflush(stderr);
91  fclose(stdout);
92  fclose(stderr);
93  }
94  exit( status );
95  }
96  else if(pid == -1)
97  {
98  perror("fork");
100  }
101  else // Parent process
102  {
103  int status = 0;
104  int childPid = waitpid(pid, &status, 0);
105  if( childPid == -1 )
106  {
107  perror("waitpid");
109  }
110  if( WIFEXITED(status) )
111  {
112  if( childPid > 0 )
113  {
114  testResult = WEXITSTATUS(status);
115  if( testResult )
116  {
117  printf("Test case %s failed: %d\n", testCase.name, testResult);
118  }
119  }
120  }
121  else if(WIFSIGNALED(status) )
122  {
123  int signal = WTERMSIG(status);
124  testResult = EXIT_STATUS_TESTCASE_ABORTED;
125  if( signal == SIGABRT )
126  {
127  printf("Test case %s failed: test case asserted\n", testCase.name );
128  }
129  else
130  {
131  printf("Test case %s failed: exit with signal %s\n", testCase.name, strsignal(WTERMSIG(status)));
132  }
133  }
134  else if(WIFSTOPPED(status))
135  {
136  printf("Test case %s failed: stopped with signal %s\n", testCase.name, strsignal(WSTOPSIG(status)));
137  }
138  }
139  fflush(stdout);
140  fflush(stderr);
141  return testResult;
142 }
143 
144 void OutputStatistics( const char* processName, int numPasses, int numFailures )
145 {
146  FILE* fp=fopen("summary.xml", "a");
147  if( fp != NULL )
148  {
149  fprintf( fp,
150  " <suite name=\"%s\">\n"
151  " <total_case>%d</total_case>\n"
152  " <pass_case>%d</pass_case>\n"
153  " <pass_rate>%5.2f</pass_rate>\n"
154  " <fail_case>%d</fail_case>\n"
155  " <fail_rate>%5.2f</fail_rate>\n"
156  " <block_case>0</block_case>\n"
157  " <block_rate>0.00</block_rate>\n"
158  " <na_case>0</na_case>\n"
159  " <na_rate>0.00</na_rate>\n"
160  " </suite>\n",
161  basename(processName),
162  numPasses+numFailures,
163  numPasses,
164  (float)numPasses/(numPasses+numFailures),
165  numFailures,
166  (float)numFailures/(numPasses+numFailures) );
167  fclose(fp);
168  }
169 }
170 
171 int RunAll( const char* processName, ::testcase tc_array[] )
172 {
173  int numFailures = 0;
174  int numPasses = 0;
175 
176  // Run test cases in child process( to kill output/handle signals ), but run serially.
177  for( unsigned int i=0; tc_array[i].name; i++)
178  {
179  int result = RunTestCaseInChildProcess( tc_array[i], false );
180  if( result == 0 )
181  {
182  numPasses++;
183  }
184  else
185  {
186  numFailures++;
187  }
188  }
189 
190  OutputStatistics( processName, numPasses, numFailures);
191 
192  return numFailures;
193 }
194 
195 // Constantly runs up to MAX_NUM_CHILDREN processes
196 int RunAllInParallel( const char* processName, ::testcase tc_array[], bool reRunFailed)
197 {
198  int numFailures = 0;
199  int numPasses = 0;
200 
201  RunningTestCases children;
202  std::vector<int> failedTestCases;
203 
204  // Fork up to MAX_NUM_CHILDREN processes, then
205  // wait. As soon as a proc completes, fork the next.
206 
207  int nextTestCase = 0;
208  int numRunningChildren = 0;
209 
210  while( tc_array[nextTestCase].name || numRunningChildren > 0)
211  {
212  // Create more children (up to the max number or til the end of the array)
213  while( numRunningChildren < MAX_NUM_CHILDREN && tc_array[nextTestCase].name )
214  {
215  int pid = fork();
216  if( pid == 0 ) // Child process
217  {
218  close(STDOUT_FILENO);
219  close(STDERR_FILENO);
220  exit( RunTestCase( tc_array[nextTestCase] ) );
221  }
222  else if(pid == -1)
223  {
224  perror("fork");
226  }
227  else // Parent process
228  {
229  TestCase tc(nextTestCase, tc_array[nextTestCase].name);
230  children[pid] = tc;
231  nextTestCase++;
232  numRunningChildren++;
233  }
234  }
235 
236  // Wait for the next child to finish
237 
238  int status=0;
239  int childPid = waitpid(-1, &status, 0);
240  if( childPid == -1 )
241  {
242  perror("waitpid");
244  }
245 
246  if( WIFEXITED(status) )
247  {
248  if( childPid > 0 )
249  {
250  int testResult = WEXITSTATUS(status);
251  if( testResult )
252  {
253  printf("Test case %s failed: %d\n", children[childPid].testCaseName, testResult);
254  failedTestCases.push_back(children[childPid].testCase);
255  numFailures++;
256  }
257  else
258  {
259  numPasses++;
260  }
261  numRunningChildren--;
262  }
263  }
264 
265  else if( WIFSIGNALED(status) || WIFSTOPPED(status))
266  {
267  status = WIFSIGNALED(status)?WTERMSIG(status):WSTOPSIG(status);
268 
269  if( childPid > 0 )
270  {
271  RunningTestCases::iterator iter = children.find(childPid);
272  if( iter != children.end() )
273  {
274  printf("Test case %s exited with signal %s\n", iter->second.testCaseName, strsignal(status));
275  failedTestCases.push_back(iter->second.testCase);
276  }
277  else
278  {
279  printf("Unknown child process: %d signaled %s\n", childPid, strsignal(status));
280  }
281 
282  numFailures++;
283  numRunningChildren--;
284  }
285  }
286  }
287 
288  OutputStatistics( processName, numPasses, numFailures );
289 
290  if( reRunFailed )
291  {
292  for( unsigned int i=0; i<failedTestCases.size(); i++)
293  {
294  char* testCaseStrapline;
295  int numChars = asprintf(&testCaseStrapline, "Test case %s", tc_array[failedTestCases[i]].name );
296  printf("\n%s\n", testCaseStrapline);
297  for(int j=0; j<numChars; j++)
298  {
299  printf("=");
300  }
301  printf("\n");
302  RunTestCaseInChildProcess( tc_array[failedTestCases[i] ], false );
303  }
304  }
305 
306  return numFailures;
307 }
308 
309 
310 
311 int FindAndRunTestCase(::testcase tc_array[], const char* testCaseName)
312 {
313  int result = EXIT_STATUS_TESTCASE_NOT_FOUND;
314 
315  for( int i = 0; tc_array[i].name; i++ )
316  {
317  if( !strcmp(testCaseName, tc_array[i].name) )
318  {
319  return RunTestCase( tc_array[i] );
320  }
321  }
322 
323  printf("Unknown testcase name: \"%s\"\n", testCaseName);
324  return result;
325 }
326 
327 void Usage(const char* program)
328 {
329  printf("Usage: \n"
330  " %s <testcase name>\t\t Execute a test case\n"
331  " %s \t\t Execute all test cases in parallel\n"
332  " %s -r\t\t Execute all test cases in parallel, rerunning failed test cases\n",
333  program, program, program);
334 }
335 
336 } // namespace
Dali Docs Home
Read more about Dali