View Javadoc

    1   /*
    2    *  soapUI, copyright (C) 2004-2007 eviware.com
    3    *
    4    *  soapUI is free software; you can redistribute it and/or modify it under the
    5    *  terms of version 2.1 of the GNU Lesser General Public License as published by
    6    *  the Free Software Foundation.
    7    *
    8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
    9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    10   *  See the GNU Lesser General Public License for more details at gnu.org.
    11   */
    12
    13  package com.eviware.soapui.tools;
    14
    15  import java.io.File;
    16  import java.io.FileOutputStream;
    17  import java.io.PrintWriter;
    18  import java.io.StringWriter;
    19  import java.util.ArrayList;
    20  import java.util.Arrays;
    21  import java.util.HashMap;
    22  import java.util.List;
    23  import java.util.Map;
    24
    25  import org.apache.commons.cli.CommandLine;
    26
    27  import com.eviware.soapui.SoapUI;
    28  import com.eviware.soapui.impl.wsdl.WsdlProject;
    29  import com.eviware.soapui.impl.wsdl.support.assertions.Assertable.AssertionStatus;
    30  import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
    31  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
    32  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
    33  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
    34  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
    35  import com.eviware.soapui.impl.wsdl.teststeps.assertions.AssertionError;
    36  import com.eviware.soapui.model.iface.Attachment;
    37  import com.eviware.soapui.model.support.PropertiesMap;
    38  import com.eviware.soapui.model.testsuite.TestCase;
    39  import com.eviware.soapui.model.testsuite.TestRunContext;
    40  import com.eviware.soapui.model.testsuite.TestRunListener;
    41  import com.eviware.soapui.model.testsuite.TestRunner;
    42  import com.eviware.soapui.model.testsuite.TestStep;
    43  import com.eviware.soapui.model.testsuite.TestStepResult;
    44  import com.eviware.soapui.model.testsuite.TestSuite;
    45  import com.eviware.soapui.model.testsuite.TestRunner.Status;
    46  import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
    47  import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
    48  import com.eviware.soapui.report.JUnitReportCollector;
    49  import com.eviware.soapui.support.Tools;
    50
    51  /***
    52   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
    53   * directly from other classes.
    54   * <p>
    55   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
    56   * calling run</p>
    57   *
    58   * @author Ole.Matzura
    59   */
    60
    61  public class SoapUITestCaseRunner extends AbstractSoapUIRunner implements TestRunListener
    62  {
    63  	public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
    64
    65  	private String testSuite;
    66  	private String testCase;
    67  	private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
    68  	private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
    69  	private List<TestCase> runningTests = new ArrayList<TestCase>();
    70  	private List<TestCase> failedTests = new ArrayList<TestCase>();
    71  	private String endpoint;
    72  	private String domain;
    73  	private String password;
    74  	private String username;
    75  	private String host;
    76
    77  	private int testSuiteCount;
    78  	private int testCaseCount;
    79  	private int testStepCount;
    80  	private int testAssertionCount;
    81
    82  	private boolean printReport;
    83  	private String outputFolder;
    84  	private boolean exportAll;
    85  	private boolean junitReport;
    86  	private int exportCount;
    87  	private JUnitReportCollector reportCollector;
    88  	private String wssPasswordType;
    89  	private WsdlProject project;
    90
    91  	/***
    92  	 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
    93  	 *
    94  	 * @param args
    95  	 * @throws Exception
    96  	 */
    97
    98  	@SuppressWarnings("static-access")
    99  	public static void main( String [] args) throws Exception
    100 	{
    101 		new SoapUITestCaseRunner().runFromCommandLine( args );
    102 	}
    103
    104 	protected boolean processCommandLine( CommandLine cmd )
    105 	{
    106 		if( cmd.hasOption( "e"))
    107 			setEndpoint( cmd.getOptionValue( "e" ) );
    108
    109 		if( cmd.hasOption( "s"))
    110 			setTestSuite( cmd.getOptionValue( "s") );
    111
    112 		if( cmd.hasOption( "c"))
    113 			setTestCase( cmd.getOptionValue( "c") );
    114
    115 		if( cmd.hasOption( "u"))
    116 			setUsername( cmd.getOptionValue( "u") );
    117
    118 		if( cmd.hasOption( "p"))
    119 			setPassword( cmd.getOptionValue( "p") );
    120
    121 		if( cmd.hasOption( "w"))
    122 			setWssPasswordType( cmd.getOptionValue( "w") );
    123
    124 		if( cmd.hasOption( "d"))
    125 			setDomain( cmd.getOptionValue( "d") );
    126
    127 		if( cmd.hasOption( "h"))
    128 			setHost( cmd.getOptionValue( "h") );
    129
    130 		if( cmd.hasOption( "f"))
    131 			setOutputFolder( cmd.getOptionValue( "f") );
    132
    133 		if( cmd.hasOption( "t"))
    134 			setSettingsFile( cmd.getOptionValue( "t" ));
    135
    136 		setEnableUI( cmd.hasOption( "i") );
    137 		setPrintReport( cmd.hasOption( "r" ) );
    138 		setExportAll( cmd.hasOption( "a" ) );
    139 		setJUnitReport( cmd.hasOption( "j" ) );
    140
    141 		return true;
    142 	}
    143
    144 	protected SoapUIOptions initCommandLineOptions()
    145 	{
    146 		SoapUIOptions options = new SoapUIOptions( "testrunner" );
    147 		options.addOption( "e", true, "Sets the endpoint" );
    148 		options.addOption( "s", true, "Sets the testsuite" );
    149 		options.addOption( "c", true, "Sets the testcase" );
    150 		options.addOption( "u", true, "Sets the username" );
    151 		options.addOption( "p", true, "Sets the password" );
    152 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
    153 		options.addOption( "i", false, "Enables Swing UI for scripts" );
    154 		options.addOption( "d", true, "Sets the domain" );
    155 		options.addOption( "h", true, "Sets the host" );
    156 		options.addOption( "r", false, "Prints a small summary report" );
    157 		options.addOption( "f", true, "Sets the output folder to export results to" );
    158 		options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
    159 		options.addOption( "a", false, "Turns on exporting of all results" );
    160 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
    161 		return options;
    162 	}
    163
    164 	/***
    165 	 * Add console appender to groovy log
    166 	 */
    167
    168
    169 	public void setExportAll(boolean exportAll)
    170 	{
    171 		this.exportAll = exportAll;
    172 	}
    173
    174 	public void setJUnitReport(boolean junitReport)
    175 	{
    176 		this.junitReport = junitReport;
    177 		if (junitReport)
    178 			reportCollector = new JUnitReportCollector();
    179 	}
    180
    181 	public void setOutputFolder(String outputFolder)
    182 	{
    183 		this.outputFolder = outputFolder;
    184 	}
    185
    186 	public SoapUITestCaseRunner()
    187 	{
    188 		super( SoapUITestCaseRunner.TITLE  );
    189 	}
    190
    191 	public SoapUITestCaseRunner( String title )
    192 	{
    193 		super( title  );
    194 	}
    195
    196 	/***
    197 	 * Controls if a short test summary should be printed after the test runs
    198 	 *
    199 	 * @param printReport a flag controlling if a summary should be printed
    200 	 */
    201
    202 	public void setPrintReport(boolean printReport)
    203 	{
    204 		this.printReport = printReport;
    205 	}
    206
    207 	/***
    208 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
    209 	 *
    210 	 * @param host the host to use by all requests
    211 	 */
    212
    213 	public void setHost(String host)
    214 	{
    215 		this.host = host;
    216 	}
    217
    218 	/***
    219 	 * Sets the domain to use for any authentications
    220 	 *
    221 	 * @param domain the domain to use for any authentications
    222 	 */
    223
    224 	public void setDomain(String domain)
    225 	{
    226 		this.domain = domain;
    227 	}
    228
    229 	/***
    230 	 * Sets the password to use for any authentications
    231 	 *
    232 	 * @param domain the password to use for any authentications
    233 	 */
    234
    235 	public void setPassword(String password)
    236 	{
    237 		this.password = password;
    238 	}
    239
    240 	/***
    241 	 * Sets the WSS password-type to use for any authentications. Setting this will result
    242 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
    243 	 * the specified username and password.
    244 	 *
    245 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
    246 	 */
    247
    248 	public void setWssPasswordType( String wssPasswordType )
    249 	{
    250 		this.wssPasswordType = wssPasswordType;
    251 	}
    252
    253 	/***
    254 	 * Sets the username to use for any authentications
    255 	 *
    256 	 * @param domain the username to use for any authentications
    257 	 */
    258
    259 	public void setUsername(String username)
    260 	{
    261 		this.username = username;
    262 	}
    263
    264 	/***
    265 	 * Runs the testcases as configured with setXXX methods
    266 	 *
    267 	 * @throws Exception thrown if any tests fail
    268 	 */
    269
    270 	public void runRunner() throws Exception
    271 	{
    272 		initGroovyLog();
    273
    274 		if( outputFolder != null )
    275 		{
    276 			//	ensure folder exists
    277 			File folder = new File( outputFolder );
    278 			if( !folder.exists())
    279 				folder.mkdirs();
    280 		}
    281
    282 		assertions.clear();
    283
    284 		String projectFile = getProjectFile();
    285
    286 		project = new WsdlProject( projectFile );
    287 		if( project.isDisabled() )
    288 			throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
    289
    290 		log.info( "Running soapUI tests in project [" + project.getName() + "]" );
    291
    292 		long startTime = System.nanoTime();
    293
    294 		// start by listening to all testcases.. (since one testcase can call another)
    295 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
    296 		{
    297 			TestSuite suite = project.getTestSuiteAt( c );
    298 			for( int i = 0; i < suite.getTestCaseCount(); i++ )
    299 			{
    300 				suite.getTestCaseAt( i ).addTestRunListener( this );
    301 				if (junitReport)
    302 					suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
    303 			}
    304 		}
    305
    306 		// now run tests..
    307 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
    308 		{
    309 			if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
    310 			{
    311 				runSuite( project.getTestSuiteAt( c ));
    312 				testSuiteCount++;
    313
    314 				//	wait for tests to finish if running in parallell mode
    315 				if( !runningTests.isEmpty() )
    316 					log.info( "Waiting for " + runningTests.size() + " tests to finish" );
    317
    318 				while( !runningTests.isEmpty() )
    319 				{
    320 					Thread.sleep( 100 );
    321 				}
    322 			}
    323 		}
    324
    325 		long timeTaken = (System.nanoTime()-startTime)/1000000;
    326
    327 		if( printReport )
    328 		{
    329 			printReport( timeTaken );
    330 		}
    331
    332 		if (junitReport) {
    333 			exportJUnitReports( reportCollector, outputFolder );
    334 		}
    335
    336 		if( assertions.size() > 0 || failedTests.size() > 0 )
    337 		{
    338 			throwFailureException();
    339 		}
    340 	}
    341
    342 	protected void throwFailureException() throws Exception
    343 	{
    344 		StringBuffer buf = new StringBuffer();
    345
    346 		for( int c = 0; c < assertions.size(); c++ )
    347 		{
    348 			WsdlMessageAssertion assertion = assertions.get( c );
    349 			WsdlTestRequest testRequest = ((WsdlTestRequest)assertion.getAssertable());
    350 			failedTests.remove( testRequest.getTestCase() );
    351
    352 			buf.append( assertion.getName() + " in [" + testRequest.getName() + "] failed;\n" );
    353 			buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
    354
    355 			WsdlTestStepResult result = assertionResults.get( assertion  );
    356 			StringWriter stringWriter = new StringWriter();
    357 			PrintWriter writer = new PrintWriter( stringWriter );
    358 			result.writeTo( writer );
    359 			buf.append( stringWriter.toString() );
    360 		}
    361
    362 		while( !failedTests.isEmpty() )
    363 		{
    364 			buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
    365 		}
    366
    367 		throw new Exception( buf.toString() );
    368 	}
    369
    370 	public void exportJUnitReports( JUnitReportCollector collector, String folder ) throws Exception
    371 	{
    372 		collector.saveReports(folder == null ? "" : folder);
    373 	}
    374
    375 	public void printReport( long timeTaken )
    376 	{
    377 		System.out.println();
    378 		System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
    379 		System.out.println( "-----------------------------" );
    380 		System.out.println( "Time Taken: " + timeTaken + "ms" );
    381 		System.out.println( "Total TestSuites: " + testSuiteCount );
    382 		System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
    383 		System.out.println( "Total TestSteps: " + testStepCount );
    384 		System.out.println( "Total Request Assertions: " + testAssertionCount );
    385 		System.out.println( "Total Failed Assertions: " + assertions.size() );
    386 		System.out.println( "Total Exported Results: " + exportCount );
    387 	}
    388
    389 	/***
    390 	 * Run tests in the specified TestSuite
    391 	 *
    392 	 * @param suite the TestSuite to run
    393 	 */
    394
    395 	public void runSuite(TestSuite suite)
    396 	{
    397 		log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
    398 		long start = System.currentTimeMillis();
    399 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
    400 		{
    401 			String name = suite.getTestCaseAt( c ).getName();
    402 			if( testCase == null || name.equalsIgnoreCase( testCase ))
    403 			{
    404 				runTestCase( suite.getTestCaseAt( c ));
    405 			}
    406 			else
    407 			{
    408 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
    409 			}
    410 		}
    411 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
    412 	}
    413
    414 	/***
    415 	 * Runs the specified TestCase
    416 	 *
    417 	 * @param testCase the testcase to run
    418 	 */
    419
    420 	private void runTestCase(TestCase testCase)
    421 	{
    422 		runningTests.add( testCase );
    423 		testCase.run( PropertiesMap.EMPTY_MAP,	testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
    424 	}
    425
    426 	/***
    427 	 * Sets the testcase to run
    428 	 *
    429 	 * @param testCase the testcase to run
    430 	 */
    431
    432 	public void setTestCase(String testCase)
    433 	{
    434       this.testCase = testCase;
    435 	}
    436
    437 	/***
    438 	 * Sets the endpoint to use for all test requests
    439 	 *
    440 	 * @param endpoint the endpoint to use for all test requests
    441 	 */
    442
    443 	public void setEndpoint(String endpoint)
    444 	{
    445 		this.endpoint = endpoint.trim();
    446 	}
    447
    448 	/***
    449 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
    450 	 *
    451 	 * @param testSuite the testSuite to run.
    452 	 */
    453
    454 	public void setTestSuite(String testSuite)
    455 	{
    456 		this.testSuite = testSuite;
    457 	}
    458
    459 	public void beforeRun(TestRunner testRunner, TestRunContext runContext)
    460 	{
    461 		log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
    462 	}
    463
    464 	public void beforeStep(TestRunner testRunner, TestRunContext runContext)
    465 	{
    466 		TestStep currentStep = runContext.getCurrentStep();
    467 		log.info( "running step [" + currentStep.getName() + "]" );
    468
    469 		if( currentStep instanceof WsdlTestRequestStep )
    470 		{
    471 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
    472 			if( endpoint != null && endpoint.length() > 0 )
    473 			{
    474 				requestStep.getTestRequest().setEndpoint( endpoint );
    475 			}
    476
    477 			if( host != null && host.length() > 0 )
    478 			{
    479 				try
    480 				{
    481 					String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
    482 					requestStep.getTestRequest().setEndpoint( ep );
    483 				}
    484 				catch (Exception e)
    485 				{
    486 					log.error( "Failed to set host on endpoint", e );
    487 				}
    488 			}
    489
    490 			if( username != null && username.length() > 0 )
    491 			{
    492 				requestStep.getTestRequest().setUsername( username );
    493 			}
    494
    495 			if( password != null && password.length() > 0 )
    496 			{
    497 				requestStep.getTestRequest().setPassword( password );
    498 			}
    499
    500 			if( domain != null && domain.length() > 0 )
    501 			{
    502 				requestStep.getTestRequest().setDomain( domain );
    503 			}
    504
    505 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
    506 			{
    507 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
    508 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
    509 			}
    510 		}
    511 	}
    512
    513 	public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
    514 	{
    515 		TestStep currentStep = runContext.getCurrentStep();
    516
    517 		if( currentStep instanceof WsdlTestRequestStep )
    518 		{
    519 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
    520 			for( int c = 0; c < requestStep.getAssertionCount(); c++ )
    521 			{
    522 				WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
    523 				log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
    524 				if( assertion.getStatus() == AssertionStatus.FAILED )
    525 				{
    526 					for( AssertionError error : assertion.getErrors())
    527 						log.error( "ASSERTION FAILED -> " + error.getMessage());
    528
    529 					assertions.add( assertion );
    530 					assertionResults.put( assertion, ( WsdlTestStepResult ) result );
    531 				}
    532
    533 				testAssertionCount++;
    534 			}
    535 		}
    536
    537 		String countPropertyName = currentStep.getName() + " run count";
    538 		Long count = (Long) runContext.getProperty( countPropertyName );
    539 		if( count == null )
    540 		{
    541 			count = new Long( 0 );
    542 		}
    543
    544 		runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
    545
    546 		if( result.getStatus() == TestStepStatus.FAILED || exportAll )
    547 		{
    548 			try
    549 			{
    550 				String nameBase = currentStep.getTestCase().getTestSuite().getName() + "-"
    551 														  + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +
    552 														  count.longValue() + "-" + result.getStatus();
    553
    554 				String fileName = nameBase + ".txt";
    555 				if( outputFolder != null )
    556 				{
    557 					fileName = outputFolder + File.separator + fileName;
    558 				}
    559
    560 				if( result.getStatus() == TestStepStatus.FAILED )
    561 					log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
    562
    563 				PrintWriter writer = new PrintWriter(fileName);
    564 				result.writeTo(writer);
    565 				writer.close();
    566
    567 				// write attachments
    568 				if( result instanceof WsdlTestRequestStepResult )
    569 				{
    570 					Attachment [] attachments = ((WsdlTestRequestStepResult)result).getResponseAttachments();
    571 					if( attachments != null && attachments.length > 0 )
    572 					{
    573 						for( int c = 0; c < attachments.length; c++ )
    574 						{
    575 							fileName = nameBase + "-attachment-" + (c+1) + ".";
    576
    577 							Attachment attachment = attachments[c];
    578 							String contentType = attachment.getContentType();
    579 							if( !"application/octet-stream".equals( contentType ) && contentType != null && contentType.indexOf( '/' ) != -1 )
    580 							{
    581 								fileName += contentType.substring( contentType.lastIndexOf( '/' )+1 );
    582 							}
    583 							else
    584 							{
    585 								fileName += "dat";
    586 							}
    587
    588 							if( outputFolder != null )
    589 							{
    590 								fileName = outputFolder + File.separator + fileName;
    591 							}
    592
    593 							FileOutputStream outFile = new FileOutputStream( fileName );
    594 							Tools.writeAll( outFile, attachment.getInputStream() );
    595 							outFile.close();
    596 						}
    597 					}
    598 				}
    599
    600 				exportCount++;
    601 			}
    602 			catch (Exception e)
    603 			{
    604 				log.error( "Error saving failed result: " + e, e );
    605 			}
    606 		}
    607
    608 		testStepCount++;
    609 	}
    610
    611 	public void afterRun(TestRunner testRunner, TestRunContext runContext)
    612 	{
    613 		log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " +
    614 				testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
    615
    616 		if( testRunner.getStatus() == Status.FAILED )
    617 		{
    618 			failedTests.add( testRunner.getTestCase() );
    619 		}
    620
    621 		runningTests.remove( testRunner.getTestCase() );
    622
    623 		testCaseCount++;
    624 	}
    625
    626 	protected WsdlProject getProject()
    627 	{
    628 		return project;
    629 	}
    630 }