View Javadoc

    1   /*
    2    *  soapUI, copyright (C) 2004-2007 eviware.com
    3    *
    4    *  soapUI is free software; you can redistribute it and/or modify it under the
    5    *  terms of version 2.1 of the GNU Lesser General Public License as published by
    6    *  the Free Software Foundation.
    7    *
    8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
    9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    10   *  See the GNU Lesser General Public License for more details at gnu.org.
    11   */
    12
    13  package com.eviware.soapui.tools;
    14
    15  import java.io.File;
    16  import java.io.FileOutputStream;
    17  import java.io.PrintWriter;
    18  import java.io.StringWriter;
    19  import java.util.ArrayList;
    20  import java.util.Arrays;
    21  import java.util.HashMap;
    22  import java.util.List;
    23  import java.util.Map;
    24
    25  import org.apache.commons.cli.CommandLine;
    26
    27  import com.eviware.soapui.SoapUI;
    28  import com.eviware.soapui.impl.wsdl.WsdlProject;
    29  import com.eviware.soapui.impl.wsdl.support.assertions.Assertable.AssertionStatus;
    30  import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
    31  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
    32  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
    33  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
    34  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
    35  import com.eviware.soapui.model.iface.Attachment;
    36  import com.eviware.soapui.model.support.PropertiesMap;
    37  import com.eviware.soapui.model.testsuite.TestCase;
    38  import com.eviware.soapui.model.testsuite.TestRunContext;
    39  import com.eviware.soapui.model.testsuite.TestRunListener;
    40  import com.eviware.soapui.model.testsuite.TestRunner;
    41  import com.eviware.soapui.model.testsuite.TestStep;
    42  import com.eviware.soapui.model.testsuite.TestStepResult;
    43  import com.eviware.soapui.model.testsuite.TestSuite;
    44  import com.eviware.soapui.model.testsuite.TestRunner.Status;
    45  import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
    46  import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
    47  import com.eviware.soapui.report.JUnitReportCollector;
    48  import com.eviware.soapui.support.Tools;
    49
    50  /***
    51   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
    52   * directly from other classes.
    53   * <p>
    54   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
    55   * calling run</p>
    56   *
    57   * @author Ole.Matzura
    58   */
    59
    60  public class SoapUITestCaseRunner extends AbstractSoapUIRunner implements TestRunListener
    61  {
    62  	public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
    63
    64  	private String testSuite;
    65  	private String testCase;
    66  	private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
    67  	private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
    68  	private List<TestCase> runningTests = new ArrayList<TestCase>();
    69  	private List<TestCase> failedTests = new ArrayList<TestCase>();
    70  	private String endpoint;
    71  	private String domain;
    72  	private String password;
    73  	private String username;
    74  	private String host;
    75
    76  	private int testSuiteCount;
    77  	private int testCaseCount;
    78  	private int testStepCount;
    79  	private int testAssertionCount;
    80
    81  	private boolean printReport;
    82  	private String outputFolder;
    83  	private boolean exportAll;
    84  	private boolean junitReport;
    85  	private int exportCount;
    86  	private JUnitReportCollector reportCollector;
    87  	private String wssPasswordType;
    88  	private WsdlProject project;
    89
    90  	/***
    91  	 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
    92  	 *
    93  	 * @param args
    94  	 * @throws Exception
    95  	 */
    96
    97  	@SuppressWarnings("static-access")
    98  	public static void main( String [] args) throws Exception
    99  	{
    100 		new SoapUITestCaseRunner().runFromCommandLine( args );
    101 	}
    102
    103 	protected boolean processCommandLine( CommandLine cmd )
    104 	{
    105 		if( cmd.hasOption( "e"))
    106 			setEndpoint( cmd.getOptionValue( "e" ) );
    107
    108 		if( cmd.hasOption( "s"))
    109 			setTestSuite( cmd.getOptionValue( "s") );
    110
    111 		if( cmd.hasOption( "c"))
    112 			setTestCase( cmd.getOptionValue( "c") );
    113
    114 		if( cmd.hasOption( "u"))
    115 			setUsername( cmd.getOptionValue( "u") );
    116
    117 		if( cmd.hasOption( "p"))
    118 			setPassword( cmd.getOptionValue( "p") );
    119
    120 		if( cmd.hasOption( "w"))
    121 			setWssPasswordType( cmd.getOptionValue( "w") );
    122
    123 		if( cmd.hasOption( "d"))
    124 			setDomain( cmd.getOptionValue( "d") );
    125
    126 		if( cmd.hasOption( "h"))
    127 			setHost( cmd.getOptionValue( "h") );
    128
    129 		if( cmd.hasOption( "f"))
    130 			setOutputFolder( cmd.getOptionValue( "f") );
    131
    132 		if( cmd.hasOption( "t"))
    133 			SoapUI.initSettings( cmd.getOptionValue( "t" ));
    134
    135 		if( cmd.hasOption( "i"))
    136 		{
    137 			enableSwingUI();
    138 		}
    139
    140 		setPrintReport( cmd.hasOption( "r" ) );
    141 		setExportAll( cmd.hasOption( "a" ) );
    142 		setJUnitReport( cmd.hasOption( "j" ) );
    143
    144 		return true;
    145 	}
    146
    147 	protected SoapUIOptions initCommandLineOptions()
    148 	{
    149 		SoapUIOptions options = new SoapUIOptions( "testrunner" );
    150 		options.addOption( "e", true, "Sets the endpoint" );
    151 		options.addOption( "s", true, "Sets the testsuite" );
    152 		options.addOption( "c", true, "Sets the testcase" );
    153 		options.addOption( "u", true, "Sets the username" );
    154 		options.addOption( "p", true, "Sets the password" );
    155 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
    156 		options.addOption( "i", false, "Enables Swing UI for scripts" );
    157 		options.addOption( "d", true, "Sets the domain" );
    158 		options.addOption( "h", true, "Sets the host" );
    159 		options.addOption( "r", false, "Prints a small summary report" );
    160 		options.addOption( "f", true, "Sets the output folder to export results to" );
    161 		options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
    162 		options.addOption( "a", false, "Turns on exporting of all results" );
    163 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
    164 		return options;
    165 	}
    166
    167 	/***
    168 	 * Add console appender to groovy log
    169 	 */
    170
    171
    172 	public void setExportAll(boolean exportAll)
    173 	{
    174 		this.exportAll = exportAll;
    175 	}
    176
    177 	public void setJUnitReport(boolean junitReport)
    178 	{
    179 		this.junitReport = junitReport;
    180 		if (junitReport)
    181 			reportCollector = new JUnitReportCollector();
    182 	}
    183
    184 	public void setOutputFolder(String outputFolder)
    185 	{
    186 		this.outputFolder = outputFolder;
    187 	}
    188
    189 	public SoapUITestCaseRunner()
    190 	{
    191 		super( SoapUITestCaseRunner.TITLE  );
    192 	}
    193
    194 	public SoapUITestCaseRunner( String title )
    195 	{
    196 		super( title  );
    197 	}
    198
    199 	/***
    200 	 * Controls if a short test summary should be printed after the test runs
    201 	 *
    202 	 * @param printReport a flag controlling if a summary should be printed
    203 	 */
    204
    205 	public void setPrintReport(boolean printReport)
    206 	{
    207 		this.printReport = printReport;
    208 	}
    209
    210 	/***
    211 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
    212 	 *
    213 	 * @param host the host to use by all requests
    214 	 */
    215
    216 	public void setHost(String host)
    217 	{
    218 		this.host = host;
    219 	}
    220
    221 	/***
    222 	 * Sets the domain to use for any authentications
    223 	 *
    224 	 * @param domain the domain to use for any authentications
    225 	 */
    226
    227 	public void setDomain(String domain)
    228 	{
    229 		log.info( "Setting domain to [" + domain + "]" );
    230 		this.domain = domain;
    231 	}
    232
    233 	/***
    234 	 * Sets the password to use for any authentications
    235 	 *
    236 	 * @param domain the password to use for any authentications
    237 	 */
    238
    239 	public void setPassword(String password)
    240 	{
    241 		log.info( "Setting password to [" + password + "]" );
    242 		this.password = password;
    243 	}
    244
    245 	/***
    246 	 * Sets the WSS password-type to use for any authentications. Setting this will result
    247 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
    248 	 * the specified username and password.
    249 	 *
    250 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
    251 	 */
    252
    253 	public void setWssPasswordType( String wssPasswordType )
    254 	{
    255 		this.wssPasswordType = wssPasswordType;
    256 	}
    257
    258 	/***
    259 	 * Sets the username to use for any authentications
    260 	 *
    261 	 * @param domain the username to use for any authentications
    262 	 */
    263
    264 	public void setUsername(String username)
    265 	{
    266 		log.info( "Setting username to [" + username + "]" );
    267 		this.username = username;
    268 	}
    269
    270 	/***
    271 	 * Runs the testcases as configured with setXXX methods
    272 	 *
    273 	 * @throws Exception thrown if any tests fail
    274 	 */
    275
    276 	public void run() throws Exception
    277 	{
    278 		initGroovyLog();
    279
    280 		if( outputFolder != null )
    281 		{
    282 			//	ensure folder exists
    283 			File folder = new File( outputFolder );
    284 			if( !folder.exists())
    285 				folder.mkdirs();
    286 		}
    287
    288 		assertions.clear();
    289
    290 		String projectFile = getProjectFile();
    291
    292 		if( !new File( projectFile ).exists() )
    293 			throw new Exception( "soapUI project file [" + projectFile + "] not found" );
    294
    295 		project = new WsdlProject( projectFile, null );
    296 		log.info( "Running soapUI tests in project [" + project.getName() + "]" );
    297
    298 		long startTime = System.nanoTime();
    299
    300 		// start by listening to all testcases.. (since one testcase can call another)
    301 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
    302 		{
    303 			TestSuite suite = project.getTestSuiteAt( c );
    304 			for( int i = 0; i < suite.getTestCaseCount(); i++ )
    305 			{
    306 				suite.getTestCaseAt( i ).addTestRunListener( this );
    307 				if (junitReport)
    308 					suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
    309 			}
    310 		}
    311
    312 		// now run tests..
    313 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
    314 		{
    315 			if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
    316 			{
    317 				runSuite( project.getTestSuiteAt( c ));
    318 				testSuiteCount++;
    319
    320 				//	wait for tests to finish if running in parallell mode
    321 				if( !runningTests.isEmpty() )
    322 					log.info( "Waiting for " + runningTests.size() + " tests to finish" );
    323
    324 				while( !runningTests.isEmpty() )
    325 				{
    326 					Thread.sleep( 100 );
    327 				}
    328 			}
    329 		}
    330
    331 		long timeTaken = (System.nanoTime()-startTime)/1000000;
    332
    333 		if( printReport )
    334 		{
    335 			printReport( timeTaken );
    336 		}
    337
    338 		if (junitReport) {
    339 			exportJUnitReports( reportCollector, outputFolder );
    340 		}
    341
    342 		if( assertions.size() > 0 || failedTests.size() > 0 )
    343 		{
    344 			throwFailureException();
    345 		}
    346 	}
    347
    348 	protected void throwFailureException() throws Exception
    349 	{
    350 		StringBuffer buf = new StringBuffer();
    351
    352 		for( int c = 0; c < assertions.size(); c++ )
    353 		{
    354 			WsdlMessageAssertion assertion = assertions.get( c );
    355 			WsdlTestRequest testRequest = ((WsdlTestRequest)assertion.getAssertable());
    356 			failedTests.remove( testRequest.getTestCase() );
    357
    358 			buf.append( assertion.getName() + " in [" + testRequest.getName() + "] failed;\n" );
    359 			buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
    360
    361 			WsdlTestStepResult result = assertionResults.get( assertion  );
    362 			StringWriter stringWriter = new StringWriter();
    363 			PrintWriter writer = new PrintWriter( stringWriter );
    364 			result.writeTo( writer );
    365 			buf.append( stringWriter.toString() );
    366 		}
    367
    368 		while( !failedTests.isEmpty() )
    369 		{
    370 			buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
    371 		}
    372
    373 		throw new Exception( buf.toString() );
    374 	}
    375
    376 	public void exportJUnitReports( JUnitReportCollector collector, String folder ) throws Exception
    377 	{
    378 		collector.saveReports(folder == null ? "" : folder);
    379 	}
    380
    381 	public void printReport( long timeTaken )
    382 	{
    383 		System.out.println();
    384 		System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
    385 		System.out.println( "-----------------------------" );
    386 		System.out.println( "Time Taken: " + timeTaken + "ms" );
    387 		System.out.println( "Total TestSuites: " + testSuiteCount );
    388 		System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
    389 		System.out.println( "Total TestSteps: " + testStepCount );
    390 		System.out.println( "Total Request Assertions: " + testAssertionCount );
    391 		System.out.println( "Total Failed Assertions: " + assertions.size() );
    392 		System.out.println( "Total Exported Results: " + exportCount );
    393 	}
    394
    395 	/***
    396 	 * Run tests in the specified TestSuite
    397 	 *
    398 	 * @param suite the TestSuite to run
    399 	 */
    400
    401 	public void runSuite(TestSuite suite)
    402 	{
    403 		log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
    404 		long start = System.currentTimeMillis();
    405 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
    406 		{
    407 			String name = suite.getTestCaseAt( c ).getName();
    408 			if( testCase == null || name.equalsIgnoreCase( testCase ))
    409 			{
    410 				runTestCase( suite.getTestCaseAt( c ));
    411 			}
    412 			else
    413 			{
    414 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
    415 			}
    416 		}
    417 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
    418 	}
    419
    420 	/***
    421 	 * Runs the specified TestCase
    422 	 *
    423 	 * @param testCase the testcase to run
    424 	 */
    425
    426 	private void runTestCase(TestCase testCase)
    427 	{
    428 		runningTests.add( testCase );
    429 		testCase.run( PropertiesMap.EMPTY_MAP,	testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
    430 	}
    431
    432 	/***
    433 	 * Sets the testcase to run
    434 	 *
    435 	 * @param testCase the testcase to run
    436 	 */
    437
    438 	public void setTestCase(String testCase)
    439 	{
    440 		log.info( "setting testCase to [" + testCase + "]" );
    441       this.testCase = testCase;
    442 	}
    443
    444 	/***
    445 	 * Sets the endpoint to use for all test requests
    446 	 *
    447 	 * @param endpoint the endpoint to use for all test requests
    448 	 */
    449
    450 	public void setEndpoint(String endpoint)
    451 	{
    452 		log.info( "setting test endpoint to [" + endpoint+ "]" );
    453 		this.endpoint = endpoint.trim();
    454 	}
    455
    456 	/***
    457 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
    458 	 *
    459 	 * @param testSuite the testSuite to run.
    460 	 */
    461
    462 	public void setTestSuite(String testSuite)
    463 	{
    464 	   log.info( "setting testSuite to [" + testSuite + "]" );
    465 		this.testSuite = testSuite;
    466 	}
    467
    468 	public void beforeRun(TestRunner testRunner, TestRunContext runContext)
    469 	{
    470 		log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
    471 	}
    472
    473 	public void beforeStep(TestRunner testRunner, TestRunContext runContext)
    474 	{
    475 		TestStep currentStep = runContext.getCurrentStep();
    476 		log.info( "running step [" + currentStep.getName() + "]" );
    477
    478 		if( currentStep instanceof WsdlTestRequestStep )
    479 		{
    480 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
    481 			if( endpoint != null && endpoint.length() > 0 )
    482 			{
    483 				requestStep.getTestRequest().setEndpoint( endpoint );
    484 			}
    485
    486 			if( host != null && host.length() > 0 )
    487 			{
    488 				try
    489 				{
    490 					String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
    491 					requestStep.getTestRequest().setEndpoint( ep );
    492 				}
    493 				catch (Exception e)
    494 				{
    495 					log.error( "Failed to set host on endpoint", e );
    496 				}
    497 			}
    498
    499 			if( username != null && username.length() > 0 )
    500 			{
    501 				requestStep.getTestRequest().setUsername( username );
    502 			}
    503
    504 			if( password != null && password.length() > 0 )
    505 			{
    506 				requestStep.getTestRequest().setPassword( password );
    507 			}
    508
    509 			if( domain != null && domain.length() > 0 )
    510 			{
    511 				requestStep.getTestRequest().setDomain( domain );
    512 			}
    513
    514 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
    515 			{
    516 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
    517 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
    518 			}
    519 		}
    520 	}
    521
    522 	public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
    523 	{
    524 		TestStep currentStep = runContext.getCurrentStep();
    525
    526 		if( currentStep instanceof WsdlTestRequestStep )
    527 		{
    528 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
    529 			for( int c = 0; c < requestStep.getAssertionCount(); c++ )
    530 			{
    531 				WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
    532 				log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
    533 				if( assertion.getStatus() == AssertionStatus.FAILED )
    534 				{
    535 					log.info( "ASSERTION FAILED -> " + assertion.getErrors());
    536 					assertions.add( assertion );
    537 					assertionResults.put( assertion, ( WsdlTestStepResult ) result );
    538 				}
    539
    540 				testAssertionCount++;
    541 			}
    542 		}
    543
    544 		String countPropertyName = currentStep.getName() + " run count";
    545 		Long count = (Long) runContext.getProperty( countPropertyName );
    546 		if( count == null )
    547 		{
    548 			count = new Long( 0 );
    549 		}
    550
    551 		runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
    552
    553 		if( result.getStatus() == TestStepStatus.FAILED || exportAll )
    554 		{
    555 			try
    556 			{
    557 				String nameBase = currentStep.getTestCase().getTestSuite().getName() + "-"
    558 														  + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +
    559 														  count.longValue() + "-" + result.getStatus();
    560
    561 				String fileName = nameBase + ".txt";
    562 				if( outputFolder != null )
    563 				{
    564 					fileName = outputFolder + File.separator + fileName;
    565 				}
    566
    567 				if( result.getStatus() == TestStepStatus.FAILED )
    568 					log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
    569
    570 				PrintWriter writer = new PrintWriter(fileName);
    571 				result.writeTo(writer);
    572 				writer.close();
    573
    574 				// write attachments
    575 				if( result instanceof WsdlTestRequestStepResult )
    576 				{
    577 					Attachment [] attachments = ((WsdlTestRequestStepResult)result).getResponseAttachments();
    578 					if( attachments != null && attachments.length > 0 )
    579 					{
    580 						for( int c = 0; c < attachments.length; c++ )
    581 						{
    582 							fileName = nameBase + "-attachment-" + (c+1) + ".";
    583
    584 							Attachment attachment = attachments[c];
    585 							String contentType = attachment.getContentType();
    586 							if( !"application/octet-stream".equals( contentType ) && contentType != null && contentType.indexOf( '/' ) != -1 )
    587 							{
    588 								fileName += contentType.substring( contentType.lastIndexOf( '/' )+1 );
    589 							}
    590 							else
    591 							{
    592 								fileName += "dat";
    593 							}
    594
    595 							if( outputFolder != null )
    596 							{
    597 								fileName = outputFolder + File.separator + fileName;
    598 							}
    599
    600 							FileOutputStream outFile = new FileOutputStream( fileName );
    601 							Tools.writeAll( outFile, attachment.getInputStream() );
    602 							outFile.close();
    603 						}
    604 					}
    605 				}
    606
    607 				exportCount++;
    608 			}
    609 			catch (Exception e)
    610 			{
    611 				log.error( "Error saving failed result: " + e, e );
    612 			}
    613 		}
    614
    615 		testStepCount++;
    616 	}
    617
    618 	public void afterRun(TestRunner testRunner, TestRunContext runContext)
    619 	{
    620 		log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " +
    621 				testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
    622
    623 		if( testRunner.getStatus() == Status.FAILED )
    624 		{
    625 			failedTests.add( testRunner.getTestCase() );
    626 		}
    627
    628 		runningTests.remove( testRunner.getTestCase() );
    629
    630 		testCaseCount++;
    631 	}
    632
    633 	protected WsdlProject getProject()
    634 	{
    635 		return project;
    636 	}
    637 }