View Javadoc

    1   /*
    2    *  soapUI, copyright (C) 2004-2007 eviware.com
    3    *
    4    *  soapUI is free software; you can redistribute it and/or modify it under the
    5    *  terms of version 2.1 of the GNU Lesser General Public License as published by
    6    *  the Free Software Foundation.
    7    *
    8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
    9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    10   *  See the GNU Lesser General Public License for more details at gnu.org.
    11   */
    12
    13  package com.eviware.soapui.tools;
    14
    15  import java.io.File;
    16  import java.io.IOException;
    17  import java.util.ArrayList;
    18  import java.util.List;
    19
    20  import org.apache.commons.cli.CommandLine;
    21
    22  import com.eviware.soapui.SoapUI;
    23  import com.eviware.soapui.impl.wsdl.WsdlProject;
    24  import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
    25  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
    26  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
    27  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
    28  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
    29  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
    30  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
    31  import com.eviware.soapui.model.testsuite.LoadTestRunContext;
    32  import com.eviware.soapui.model.testsuite.LoadTestRunListener;
    33  import com.eviware.soapui.model.testsuite.LoadTestRunner;
    34  import com.eviware.soapui.model.testsuite.TestCase;
    35  import com.eviware.soapui.model.testsuite.TestRunContext;
    36  import com.eviware.soapui.model.testsuite.TestRunner;
    37  import com.eviware.soapui.model.testsuite.TestStep;
    38  import com.eviware.soapui.model.testsuite.TestStepResult;
    39  import com.eviware.soapui.model.testsuite.TestSuite;
    40  import com.eviware.soapui.settings.UISettings;
    41  import com.eviware.soapui.support.SoapUIException;
    42  import com.eviware.soapui.support.Tools;
    43
    44  /***
    45   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
    46   * directly from other classes.
    47   * <p>
    48   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
    49   * calling run</p>
    50   *
    51   * @author Ole.Matzura
    52   */
    53
    54  public class SoapUILoadTestRunner extends AbstractSoapUIRunner implements LoadTestRunListener
    55  {
    56  	private String testSuite;
    57  	private String testCase;
    58  	private String endpoint;
    59  	private String domain;
    60  	private String password;
    61  	private String username;
    62  	private String host;
    63  	private String loadTest;
    64  	private boolean printReport;
    65  	private String outputFolder;
    66  	private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
    67  	private int testCaseCount;
    68  	private int loadTestCount;
    69  	private int limit = -1;
    70  	private String wssPasswordType;
    71
    72  	public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
    73
    74  	/***
    75  	 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
    76  	 *
    77  	 * @param args
    78  	 * @throws Exception
    79  	 */
    80
    81  	public static void main( String [] args )
    82  	{
    83  		new SoapUILoadTestRunner().runFromCommandLine( args );
    84  	}
    85
    86  	protected boolean processCommandLine( CommandLine cmd )
    87  	{
    88  		if( cmd.hasOption( "e"))
    89  			setEndpoint( cmd.getOptionValue( "e" ) );
    90
    91  		if( cmd.hasOption( "s"))
    92  			setTestSuite( cmd.getOptionValue( "s") );
    93
    94  		if( cmd.hasOption( "c"))
    95  			setTestCase( cmd.getOptionValue( "c") );
    96
    97  		if( cmd.hasOption( "l"))
    98  			setLoadTest( cmd.getOptionValue( "l") );
    99
    100 		if( cmd.hasOption( "u"))
    101 			setUsername( cmd.getOptionValue( "u") );
    102
    103 		if( cmd.hasOption( "p"))
    104 			setPassword( cmd.getOptionValue( "p") );
    105
    106 		if( cmd.hasOption( "w"))
    107 			setWssPasswordType( cmd.getOptionValue( "w") );
    108
    109 		if( cmd.hasOption( "d"))
    110 			setDomain( cmd.getOptionValue( "d") );
    111
    112 		if( cmd.hasOption( "h"))
    113 			setHost( cmd.getOptionValue( "h") );
    114
    115 		if( cmd.hasOption( "m"))
    116 			setLimit( Integer.parseInt( cmd.getOptionValue( "m") ));
    117
    118 		if( cmd.hasOption( "f"))
    119 			setOutputFolder( cmd.getOptionValue( "f") );
    120
    121 		if( cmd.hasOption( "t"))
    122 			setSettingsFile( cmd.getOptionValue( "t" ));
    123
    124 		setPrintReport( cmd.hasOption( "r") );
    125
    126 		return true;
    127 	}
    128
    129 	public void setLimit( int limit )
    130 	{
    131 		this.limit = limit;
    132 	}
    133
    134 	protected SoapUIOptions initCommandLineOptions()
    135 	{
    136 		SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
    137 		options.addOption( "e", true, "Sets the endpoint" );
    138 		options.addOption( "s", true, "Sets the testsuite" );
    139 		options.addOption( "c", true, "Sets the testcase" );
    140 		options.addOption( "l", true, "Sets the loadtest" );
    141 		options.addOption( "u", true, "Sets the username" );
    142 		options.addOption( "p", true, "Sets the password" );
    143 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
    144 		options.addOption( "d", true, "Sets the domain" );
    145 		options.addOption( "h", true, "Sets the host" );
    146 		options.addOption( "m", true, "Overrides the LoadTest limit" );
    147 		options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
    148 		options.addOption( "f", true, "Sets the output folder to export to" );
    149 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
    150 		return options;
    151 	}
    152
    153 	public SoapUILoadTestRunner()
    154 	{
    155 		this( TITLE );
    156 	}
    157
    158 	public SoapUILoadTestRunner( String title )
    159 	{
    160 		super( title );
    161 	}
    162
    163 	public void setLoadTest(String loadTest)
    164 	{
    165 		this.loadTest = loadTest;
    166 	}
    167
    168 	public void setOutputFolder(String outputFolder)
    169 	{
    170 		this.outputFolder = outputFolder;
    171 	}
    172
    173 	public void setPrintReport(boolean printReport)
    174 	{
    175 		this.printReport = printReport;
    176 	}
    177
    178 	/***
    179 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
    180 	 *
    181 	 * @param host the host to use by all requests
    182 	 */
    183
    184 	public void setHost(String host)
    185 	{
    186 		this.host = host;
    187 	}
    188
    189 	/***
    190 	 * Sets the domain to use for any authentications
    191 	 *
    192 	 * @param domain the domain to use for any authentications
    193 	 */
    194
    195 	public void setDomain(String domain)
    196 	{
    197 		log.info( "Setting domain to [" + domain + "]" );
    198 		this.domain = domain;
    199 	}
    200
    201 	/***
    202 	 * Sets the password to use for any authentications
    203 	 *
    204 	 * @param domain the password to use for any authentications
    205 	 */
    206
    207 	public void setPassword(String password)
    208 	{
    209 		log.info( "Setting password to [" + password + "]" );
    210 		this.password = password;
    211 	}
    212
    213 	/***
    214 	 * Sets the WSS password-type to use for any authentications. Setting this will result
    215 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
    216 	 * the specified username and password.
    217 	 *
    218 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
    219 	 */
    220
    221 	public void setWssPasswordType( String wssPasswordType )
    222 	{
    223 		this.wssPasswordType = wssPasswordType;
    224 	}
    225
    226 	/***
    227 	 * Sets the username to use for any authentications
    228 	 *
    229 	 * @param domain the username to use for any authentications
    230 	 */
    231
    232 	public void setUsername(String username)
    233 	{
    234 		log.info( "Setting username to [" + username + "]" );
    235 		this.username = username;
    236 	}
    237
    238 	/***
    239 	 * Runs the testcases as configured with setXXX methods
    240 	 *
    241 	 * @throws Exception thrown if any tests fail
    242 	 */
    243
    244 	public void runRunner() throws Exception
    245 	{
    246 		if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ))
    247 		{
    248 			initGroovyLog();
    249 		}
    250
    251 		String projectFile = getProjectFile();
    252
    253 		WsdlProject project = new WsdlProject( projectFile );
    254 		if( project.isDisabled() )
    255 			throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
    256 		int suiteCount = 0;
    257
    258 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
    259 		{
    260 			if( testSuite == null ||
    261 				 project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
    262 			{
    263 				runSuite( project.getTestSuiteAt( c ));
    264 				suiteCount++;
    265 			}
    266 		}
    267
    268 		if( suiteCount == 0 )
    269 		{
    270 			log.warn( "No test-suites matched argument [" + testSuite + "]" );
    271 		}
    272 		else if( testCaseCount == 0 )
    273 		{
    274 			log.warn( "No test-cases matched argument [" + testCase + "]" );
    275 		}
    276 		else if( loadTestCount == 0 )
    277 		{
    278 			log.warn( "No load-tests matched argument [" + loadTest + "]" );
    279 		}
    280 		else if( !failedTests.isEmpty() )
    281 		{
    282 			log.info( failedTests.size() + " load tests failed:" );
    283 			for( LoadTestRunner loadTestRunner : failedTests )
    284 			{
    285 				log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
    286 			}
    287
    288 			throw new SoapUIException( "LoadTests failed" );
    289 		}
    290 	}
    291
    292 	/***
    293 	 * Run tests in the specified TestSuite
    294 	 *
    295 	 * @param suite the TestSuite to run
    296 	 */
    297
    298 	public void runSuite(TestSuite suite)
    299 	{
    300 		long start = System.currentTimeMillis();
    301 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
    302 		{
    303 			String name = suite.getTestCaseAt( c ).getName();
    304 			if( testCase == null ||
    305 				 name.equalsIgnoreCase( testCase ))
    306 			{
    307 				runTestCase( suite.getTestCaseAt( c ));
    308 				testCaseCount++;
    309 			}
    310 			else
    311 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
    312 		}
    313 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
    314 	}
    315
    316 	/***
    317 	 * Runs the specified TestCase
    318 	 *
    319 	 * @param testCase the testcase to run
    320 	 */
    321
    322 	private void runTestCase(TestCase testCase)
    323 	{
    324 		for( int c = 0; c < testCase.getLoadTestCount(); c++ )
    325 		{
    326 			String name = testCase.getLoadTestAt( c ).getName();
    327 			if( loadTest == null || loadTest.equalsIgnoreCase( name ))
    328 			{
    329 				runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ));
    330 				loadTestCount++;
    331 			}
    332 		}
    333 	}
    334
    335 	/***
    336 	 * Runs the specified LoadTest
    337 	 *
    338 	 * @param loadTest the loadTest to run
    339 	 */
    340
    341 	private void runWsdlLoadTest(WsdlLoadTest loadTest)
    342 	{
    343 		try
    344 		{
    345 			log.info( "Running LoadTest [" + loadTest.getName() + "]" );
    346 			if( limit >= 0 )
    347 			{
    348 				log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
    349 				loadTest.setTestLimit( limit );
    350 			}
    351
    352 			loadTest.addLoadTestRunListener(this);
    353 			LoadTestRunner runner = loadTest.run();
    354
    355 			// wait for test to finish
    356 			while (runner.getStatus() == LoadTestRunner.Status.RUNNING )
    357 			{
    358 				log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " +
    359 						runner.getRunningThreadCount() );
    360 				Thread.sleep(1000);
    361 			}
    362
    363 			log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
    364
    365 			if( printReport )
    366 			{
    367 				log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
    368
    369 				loadTest.getStatisticsModel().finish();
    370
    371 				exportLog( loadTest );
    372 				exportStatistics( loadTest );
    373 			}
    374 		}
    375 		catch (Exception e)
    376 		{
    377 			SoapUI.logError( e );
    378 			log.error( e );
    379 		}
    380 	}
    381
    382 	private void exportStatistics(WsdlLoadTest loadTest) throws IOException
    383 	{
    384 		ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
    385 		String statisticsFileName = loadTest.getName() + "-statistics.txt";
    386 		if( outputFolder != null )
    387 		{
    388 			ensureOutputFolder();
    389 			statisticsFileName = outputFolder + File.separator + statisticsFileName;
    390 		}
    391
    392 		int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ));
    393 		log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
    394 	}
    395
    396 	private void exportLog(WsdlLoadTest loadTest) throws IOException
    397 	{
    398 		// export log first
    399 		LoadTestLog loadTestLog = loadTest.getLoadTestLog();
    400 		ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction(loadTestLog, null);
    401 		String logFileName = loadTest.getName() + "-log.txt";
    402 		if( outputFolder != null )
    403 		{
    404 			ensureOutputFolder();
    405 			logFileName = outputFolder + File.separator + logFileName;
    406 		}
    407
    408 		int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ));
    409 		log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
    410
    411 		int errorCnt = 0;
    412 		for( int c = 0; c < loadTestLog.getSize(); c++ )
    413 		{
    414 			LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
    415
    416 			if( entry != null && entry.isError() )
    417 			{
    418 				String entryFileName = loadTest.getName() + "-error-" + errorCnt++ + "-entry.txt";
    419 				if( outputFolder != null )
    420 				{
    421 					ensureOutputFolder();
    422 					entryFileName = outputFolder + File.separator + entryFileName;
    423 				}
    424
    425 				try
    426 				{
    427 					entry.exportToFile( entryFileName );
    428 				}
    429 				catch (Exception e)
    430 				{
    431 					SoapUI.logError( e );
    432 				}
    433 			}
    434 		}
    435 		log.info( "Exported " + errorCnt + " error results" );
    436 	}
    437
    438 	private void ensureOutputFolder()
    439 	{
    440 		File folder = new File( outputFolder );
    441 		if( !folder.exists())
    442 			folder.mkdirs();
    443 	}
    444
    445 	/***
    446 	 * Sets the testcase to run
    447 	 *
    448 	 * @param testCase the testcase to run
    449 	 */
    450
    451 	public void setTestCase(String testCase)
    452 	{
    453       this.testCase = testCase;
    454 	}
    455
    456 	/***
    457 	 * Sets the endpoint to use for all test requests
    458 	 *
    459 	 * @param endpoint the endpoint to use for all test requests
    460 	 */
    461
    462 	public void setEndpoint(String endpoint)
    463 	{
    464 		this.endpoint = endpoint.trim();
    465 	}
    466
    467 	/***
    468 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
    469 	 *
    470 	 * @param testSuite the testSuite to run.
    471 	 */
    472
    473 	public void setTestSuite(String testSuite)
    474 	{
    475 		this.testSuite = testSuite;
    476 	}
    477
    478 	public void afterLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
    479 	{
    480 		if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
    481 		{
    482 			failedTests.add( loadTestRunner );
    483 		}
    484 	}
    485
    486 	public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
    487 	{
    488 	}
    489
    490 	public void afterTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult)
    491 	{
    492 	}
    493
    494 	public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
    495 	{
    496 	}
    497
    498 	public void beforeTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
    499 	{
    500 	}
    501
    502 	public void beforeTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep)
    503 	{
    504 		if( testStep instanceof WsdlTestRequestStep )
    505 		{
    506 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) testStep;
    507 			if( endpoint != null && endpoint.length() > 0 )
    508 			{
    509 				requestStep.getTestRequest().setEndpoint( endpoint );
    510 			}
    511
    512 			if( host != null && host.length() > 0 )
    513 			{
    514 				try
    515 				{
    516 					String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
    517 					requestStep.getTestRequest().setEndpoint( ep );
    518 				}
    519 				catch (Exception e)
    520 				{
    521 					log.error( "Failed to set host on endpoint", e );
    522 				}
    523 			}
    524
    525 			if( username != null && username.length() > 0 )
    526 			{
    527 				requestStep.getTestRequest().setUsername( username );
    528 			}
    529
    530 			if( password != null && password.length() > 0 )
    531 			{
    532 				requestStep.getTestRequest().setPassword( password );
    533 			}
    534
    535 			if( domain != null && domain.length() > 0 )
    536 			{
    537 				requestStep.getTestRequest().setDomain( domain );
    538 			}
    539
    540 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
    541 			{
    542 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
    543 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
    544 			}
    545 		}
    546 	}
    547
    548 	public void loadTestStarted(LoadTestRunner loadTestRunner, LoadTestRunContext context)
    549 	{
    550 	}
    551
    552 	public void loadTestStopped(LoadTestRunner loadTestRunner, LoadTestRunContext context)
    553 	{
    554 	}
    555 }