View Javadoc

    1   /*
    2    *  soapUI, copyright (C) 2004-2007 eviware.com
    3    *
    4    *  soapUI is free software; you can redistribute it and/or modify it under the
    5    *  terms of version 2.1 of the GNU Lesser General Public License as published by
    6    *  the Free Software Foundation.
    7    *
    8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
    9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    10   *  See the GNU Lesser General Public License for more details at gnu.org.
    11   */
    12
    13  package com.eviware.soapui.tools;
    14
    15  import java.io.File;
    16  import java.io.IOException;
    17  import java.util.ArrayList;
    18  import java.util.List;
    19
    20  import org.apache.commons.cli.CommandLine;
    21
    22  import com.eviware.soapui.SoapUI;
    23  import com.eviware.soapui.impl.wsdl.WsdlProject;
    24  import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
    25  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
    26  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
    27  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
    28  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
    29  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
    30  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
    31  import com.eviware.soapui.model.testsuite.LoadTestRunContext;
    32  import com.eviware.soapui.model.testsuite.LoadTestRunListener;
    33  import com.eviware.soapui.model.testsuite.LoadTestRunner;
    34  import com.eviware.soapui.model.testsuite.TestCase;
    35  import com.eviware.soapui.model.testsuite.TestRunContext;
    36  import com.eviware.soapui.model.testsuite.TestRunner;
    37  import com.eviware.soapui.model.testsuite.TestStep;
    38  import com.eviware.soapui.model.testsuite.TestStepResult;
    39  import com.eviware.soapui.model.testsuite.TestSuite;
    40  import com.eviware.soapui.settings.UISettings;
    41  import com.eviware.soapui.support.SoapUIException;
    42  import com.eviware.soapui.support.Tools;
    43
    44  /***
    45   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
    46   * directly from other classes.
    47   * <p>
    48   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
    49   * calling run</p>
    50   *
    51   * @author Ole.Matzura
    52   */
    53
    54  public class SoapUILoadTestRunner extends AbstractSoapUIRunner implements LoadTestRunListener
    55  {
    56  	private String testSuite;
    57  	private String testCase;
    58  	private String endpoint;
    59
    60  	private String domain;
    61  	private String password;
    62  	private String username;
    63  	private String host;
    64  	private String loadTest;
    65  	private boolean printReport;
    66  	private String outputFolder;
    67  	private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
    68  	private int testCaseCount;
    69  	private int loadTestCount;
    70  	private String wssPasswordType;
    71
    72  	public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
    73
    74  	/***
    75  	 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
    76  	 *
    77  	 * @param args
    78  	 * @throws Exception
    79  	 */
    80
    81  	public static void main( String [] args )
    82  	{
    83  		new SoapUILoadTestRunner().runFromCommandLine( args );
    84  	}
    85
    86  	protected boolean processCommandLine( CommandLine cmd )
    87  	{
    88  		if( cmd.hasOption( "e"))
    89  			setEndpoint( cmd.getOptionValue( "e" ) );
    90
    91  		if( cmd.hasOption( "s"))
    92  			setTestSuite( cmd.getOptionValue( "s") );
    93
    94  		if( cmd.hasOption( "c"))
    95  			setTestCase( cmd.getOptionValue( "c") );
    96
    97  		if( cmd.hasOption( "l"))
    98  			setLoadTest( cmd.getOptionValue( "l") );
    99
    100 		if( cmd.hasOption( "u"))
    101 			setUsername( cmd.getOptionValue( "u") );
    102
    103 		if( cmd.hasOption( "p"))
    104 			setPassword( cmd.getOptionValue( "p") );
    105
    106 		if( cmd.hasOption( "w"))
    107 			setWssPasswordType( cmd.getOptionValue( "w") );
    108
    109 		if( cmd.hasOption( "d"))
    110 			setDomain( cmd.getOptionValue( "d") );
    111
    112 		if( cmd.hasOption( "h"))
    113 			setHost( cmd.getOptionValue( "h") );
    114
    115 		if( cmd.hasOption( "f"))
    116 			setOutputFolder( cmd.getOptionValue( "f") );
    117
    118 		if( cmd.hasOption( "t"))
    119 			SoapUI.initSettings( cmd.getOptionValue( "t" ));
    120
    121 		setPrintReport( cmd.hasOption( "r") );
    122
    123 		return true;
    124 	}
    125
    126 	protected SoapUIOptions initCommandLineOptions()
    127 	{
    128 		SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
    129 		options.addOption( "e", true, "Sets the endpoint" );
    130 		options.addOption( "s", true, "Sets the testsuite" );
    131 		options.addOption( "c", true, "Sets the testcase" );
    132 		options.addOption( "l", true, "Sets the loadtest" );
    133 		options.addOption( "u", true, "Sets the username" );
    134 		options.addOption( "p", true, "Sets the password" );
    135 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
    136 		options.addOption( "d", true, "Sets the domain" );
    137 		options.addOption( "h", true, "Sets the host" );
    138 		options.addOption( "r", false, "Exports statistics and testlogs for each loadtest run" );
    139 		options.addOption( "f", true, "Sets the output folder to export to" );
    140 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
    141 		return options;
    142 	}
    143
    144 	public SoapUILoadTestRunner()
    145 	{
    146 		super( TITLE );
    147 	}
    148
    149 	public SoapUILoadTestRunner( String title )
    150 	{
    151 		super( title );
    152 	}
    153
    154 	public void setLoadTest(String loadTest)
    155 	{
    156 		this.loadTest = loadTest;
    157 	}
    158
    159 	public void setOutputFolder(String outputFolder)
    160 	{
    161 		this.outputFolder = outputFolder;
    162 	}
    163
    164 	public void setPrintReport(boolean printReport)
    165 	{
    166 		this.printReport = printReport;
    167 	}
    168
    169 	/***
    170 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
    171 	 *
    172 	 * @param host the host to use by all requests
    173 	 */
    174
    175 	public void setHost(String host)
    176 	{
    177 		this.host = host;
    178 	}
    179
    180 	/***
    181 	 * Sets the domain to use for any authentications
    182 	 *
    183 	 * @param domain the domain to use for any authentications
    184 	 */
    185
    186 	public void setDomain(String domain)
    187 	{
    188 		log.info( "Setting domain to [" + domain + "]" );
    189 		this.domain = domain;
    190 	}
    191
    192 	/***
    193 	 * Sets the password to use for any authentications
    194 	 *
    195 	 * @param domain the password to use for any authentications
    196 	 */
    197
    198 	public void setPassword(String password)
    199 	{
    200 		log.info( "Setting password to [" + password + "]" );
    201 		this.password = password;
    202 	}
    203
    204 	/***
    205 	 * Sets the WSS password-type to use for any authentications. Setting this will result
    206 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
    207 	 * the specified username and password.
    208 	 *
    209 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
    210 	 */
    211
    212 	public void setWssPasswordType( String wssPasswordType )
    213 	{
    214 		this.wssPasswordType = wssPasswordType;
    215 	}
    216
    217 	/***
    218 	 * Sets the username to use for any authentications
    219 	 *
    220 	 * @param domain the username to use for any authentications
    221 	 */
    222
    223 	public void setUsername(String username)
    224 	{
    225 		log.info( "Setting username to [" + username + "]" );
    226 		this.username = username;
    227 	}
    228
    229 	/***
    230 	 * Runs the testcases as configured with setXXX methods
    231 	 *
    232 	 * @throws Exception thrown if any tests fail
    233 	 */
    234
    235 	public void run() throws Exception
    236 	{
    237 		if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ))
    238 		{
    239 			initGroovyLog();
    240 		}
    241
    242 		String projectFile = getProjectFile();
    243
    244 		if( !new File( projectFile ).exists() )
    245 			throw new SoapUIException( "soapUI project file [" + projectFile + "] not found" );
    246
    247 		WsdlProject project = new WsdlProject( projectFile, null );
    248 		int suiteCount = 0;
    249
    250 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
    251 		{
    252 			if( testSuite == null ||
    253 				 project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
    254 			{
    255 				runSuite( project.getTestSuiteAt( c ));
    256 				suiteCount++;
    257 			}
    258 		}
    259
    260 		if( suiteCount == 0 )
    261 		{
    262 			log.warn( "No test-suites matched argument [" + testSuite + "]" );
    263 		}
    264 		else if( testCaseCount == 0 )
    265 		{
    266 			log.warn( "No test-cases matched argument [" + testCase + "]" );
    267 		}
    268 		else if( loadTestCount == 0 )
    269 		{
    270 			log.warn( "No load-tests matched argument [" + loadTest + "]" );
    271 		}
    272 		else if( !failedTests.isEmpty() )
    273 		{
    274 			log.info( failedTests.size() + " load tests failed:" );
    275 			for( LoadTestRunner loadTestRunner : failedTests )
    276 			{
    277 				log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
    278 			}
    279
    280 			throw new SoapUIException( "LoadTests failed" );
    281 		}
    282 	}
    283
    284 	/***
    285 	 * Run tests in the specified TestSuite
    286 	 *
    287 	 * @param suite the TestSuite to run
    288 	 */
    289
    290 	public void runSuite(TestSuite suite)
    291 	{
    292 		long start = System.currentTimeMillis();
    293 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
    294 		{
    295 			String name = suite.getTestCaseAt( c ).getName();
    296 			if( testCase == null ||
    297 				 name.equalsIgnoreCase( testCase ))
    298 			{
    299 				runTestCase( suite.getTestCaseAt( c ));
    300 				testCaseCount++;
    301 			}
    302 			else
    303 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
    304 		}
    305 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
    306 	}
    307
    308 	/***
    309 	 * Runs the specified TestCase
    310 	 *
    311 	 * @param testCase the testcase to run
    312 	 */
    313
    314 	private void runTestCase(TestCase testCase)
    315 	{
    316 		for( int c = 0; c < testCase.getLoadTestCount(); c++ )
    317 		{
    318 			String name = testCase.getLoadTestAt( c ).getName();
    319 			if( loadTest == null || loadTest.equalsIgnoreCase( name ))
    320 			{
    321 				runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ));
    322 				loadTestCount++;
    323 			}
    324 		}
    325 	}
    326
    327 	/***
    328 	 * Runs the specified LoadTest
    329 	 *
    330 	 * @param loadTest the loadTest to run
    331 	 */
    332
    333 	private void runWsdlLoadTest(WsdlLoadTest loadTest)
    334 	{
    335 		try
    336 		{
    337 			log.info( "Running LoadTest [" + loadTest.getName() + "]" );
    338 			loadTest.addLoadTestRunListener(this);
    339 			LoadTestRunner runner = loadTest.run();
    340
    341 			// wait for test to finish
    342 			while (runner.getStatus() == LoadTestRunner.Status.RUNNING )
    343 			{
    344 				log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " +
    345 						runner.getRunningThreadCount() );
    346 				Thread.sleep(1000);
    347 			}
    348
    349 			log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
    350
    351 			if( printReport )
    352 			{
    353 				log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
    354
    355 				loadTest.getStatisticsModel().finish();
    356
    357 				exportLog( loadTest );
    358 				exportStatistics( loadTest );
    359 			}
    360 		}
    361 		catch (Exception e)
    362 		{
    363 			SoapUI.logError( e );
    364 			log.error( e );
    365 		}
    366 	}
    367
    368 	private void exportStatistics(WsdlLoadTest loadTest) throws IOException
    369 	{
    370 		ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
    371 		String statisticsFileName = loadTest.getName() + "-statistics.txt";
    372 		if( outputFolder != null )
    373 		{
    374 			ensureOutputFolder();
    375 			statisticsFileName = outputFolder + File.separator + statisticsFileName;
    376 		}
    377
    378 		int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ));
    379 		log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
    380 	}
    381
    382 	private void exportLog(WsdlLoadTest loadTest) throws IOException
    383 	{
    384 		// export log first
    385 		LoadTestLog loadTestLog = loadTest.getLoadTestLog();
    386 		ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction(loadTestLog);
    387 		String logFileName = loadTest.getName() + "-log.txt";
    388 		if( outputFolder != null )
    389 		{
    390 			ensureOutputFolder();
    391 			logFileName = outputFolder + File.separator + logFileName;
    392 		}
    393
    394 		int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ));
    395 		log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
    396
    397 		int errorCnt = 0;
    398 		for( int c = 0; c < loadTestLog.getSize(); c++ )
    399 		{
    400 			LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
    401
    402 			if( entry != null && entry.isError() )
    403 			{
    404 				String entryFileName = loadTest.getName() + "-error-" + errorCnt++ + "-entry.txt";
    405 				if( outputFolder != null )
    406 				{
    407 					ensureOutputFolder();
    408 					entryFileName = outputFolder + File.separator + entryFileName;
    409 				}
    410
    411 				try
    412 				{
    413 					entry.exportToFile( entryFileName );
    414 				}
    415 				catch (Exception e)
    416 				{
    417 					SoapUI.logError( e );
    418 				}
    419 			}
    420 		}
    421 		log.info( "Exported " + errorCnt + " error results" );
    422 	}
    423
    424 	private void ensureOutputFolder()
    425 	{
    426 		File folder = new File( outputFolder );
    427 		if( !folder.exists())
    428 			folder.mkdirs();
    429 	}
    430
    431 	/***
    432 	 * Sets the testcase to run
    433 	 *
    434 	 * @param testCase the testcase to run
    435 	 */
    436
    437 	public void setTestCase(String testCase)
    438 	{
    439 		log.info( "setting testCase to [" + testCase + "]" );
    440       this.testCase = testCase;
    441 	}
    442
    443 	/***
    444 	 * Sets the endpoint to use for all test requests
    445 	 *
    446 	 * @param endpoint the endpoint to use for all test requests
    447 	 */
    448
    449 	public void setEndpoint(String endpoint)
    450 	{
    451 		log.info( "setting test endpoint to [" + endpoint+ "]" );
    452 		this.endpoint = endpoint.trim();
    453 	}
    454
    455 	/***
    456 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
    457 	 *
    458 	 * @param testSuite the testSuite to run.
    459 	 */
    460
    461 	public void setTestSuite(String testSuite)
    462 	{
    463 	   log.info( "setting testSuite to [" + testSuite + "]" );
    464 		this.testSuite = testSuite;
    465 	}
    466
    467 	public void afterLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
    468 	{
    469 		if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
    470 		{
    471 			failedTests.add( loadTestRunner );
    472 		}
    473 	}
    474
    475 	public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
    476 	{
    477 	}
    478
    479 	public void afterTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult)
    480 	{
    481 	}
    482
    483 	public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
    484 	{
    485 	}
    486
    487 	public void beforeTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
    488 	{
    489 	}
    490
    491 	public void beforeTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep)
    492 	{
    493 		if( testStep instanceof WsdlTestRequestStep )
    494 		{
    495 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) testStep;
    496 			if( endpoint != null && endpoint.length() > 0 )
    497 			{
    498 				requestStep.getTestRequest().setEndpoint( endpoint );
    499 			}
    500
    501 			if( host != null && host.length() > 0 )
    502 			{
    503 				try
    504 				{
    505 					String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
    506 					requestStep.getTestRequest().setEndpoint( ep );
    507 				}
    508 				catch (Exception e)
    509 				{
    510 					log.error( "Failed to set host on endpoint", e );
    511 				}
    512 			}
    513
    514 			if( username != null && username.length() > 0 )
    515 			{
    516 				requestStep.getTestRequest().setUsername( username );
    517 			}
    518
    519 			if( password != null && password.length() > 0 )
    520 			{
    521 				requestStep.getTestRequest().setPassword( password );
    522 			}
    523
    524 			if( domain != null && domain.length() > 0 )
    525 			{
    526 				requestStep.getTestRequest().setDomain( domain );
    527 			}
    528
    529 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
    530 			{
    531 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
    532 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
    533 			}
    534 		}
    535 	}
    536
    537 	public void loadTestStarted(LoadTestRunner loadTestRunner, LoadTestRunContext context)
    538 	{
    539 	}
    540
    541 	public void loadTestStopped(LoadTestRunner loadTestRunner, LoadTestRunContext context)
    542 	{
    543 	}
    544 }