connectionPair) {
+ return System.currentTimeMillis() - connectionPair.second >= SESSION_DURATION;
+ }
+
+ private String generateStatementKey(String threadName, String tableName) {
+ return threadName + "_" + tableName;
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/DSQLTPCCLoader.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/DSQLTPCCLoader.java
index 238072d..0e31800 100644
--- a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/DSQLTPCCLoader.java
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/DSQLTPCCLoader.java
@@ -17,1093 +17,348 @@
package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql;
+import com.google.common.base.Preconditions;
import com.oltpbenchmark.api.Loader;
import com.oltpbenchmark.api.LoaderThread;
import com.oltpbenchmark.benchmarks.tpcc.TPCCBenchmark;
-import com.oltpbenchmark.benchmarks.tpcc.TPCCConfig;
-import com.oltpbenchmark.benchmarks.tpcc.TPCCConstants;
-import com.oltpbenchmark.benchmarks.tpcc.TPCCUtil;
-import com.oltpbenchmark.benchmarks.tpcc.pojo.Customer;
-import com.oltpbenchmark.benchmarks.tpcc.pojo.District;
-import com.oltpbenchmark.benchmarks.tpcc.pojo.History;
-import com.oltpbenchmark.benchmarks.tpcc.pojo.Item;
-import com.oltpbenchmark.benchmarks.tpcc.pojo.NewOrder;
-import com.oltpbenchmark.benchmarks.tpcc.pojo.Oorder;
-import com.oltpbenchmark.benchmarks.tpcc.pojo.OrderLine;
-import com.oltpbenchmark.benchmarks.tpcc.pojo.Stock;
-import com.oltpbenchmark.benchmarks.tpcc.pojo.Warehouse;
-import com.oltpbenchmark.catalog.Table;
-import com.oltpbenchmark.util.ConnectionUtil;
-import com.oltpbenchmark.util.SQLUtil;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders.CustomerTableLoader;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders.DistrictTableLoader;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders.HistoryTableLoader;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders.ItemTableLoader;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders.NewOrderTableLoader;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders.OrderLineTableLoader;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders.OrderTableLoader;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders.StockTableLoader;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders.WarehouseTableLoader;
+import com.oltpbenchmark.types.DatabaseType;
import java.sql.Connection;
import java.sql.PreparedStatement;
+import java.sql.ResultSet;
import java.sql.SQLException;
-import java.sql.Statement;
-import java.sql.Timestamp;
-import java.sql.Types;
import java.util.ArrayList;
import java.util.List;
-import java.util.Random;
import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Consumer;
-/** Custom DSQL TPC-C Benchmark Loader */
-/*
- * Modifications:
- * - Auto retry with a new connection when a thread loading data
- * into tables fails (set maxRetries in config.xml). This helps
- * avoid any OCC errors. (OC001)
- * - Don't fail on duplicate key exceptions. Move to next statement.
- * - Run ANALYZE on tables after loading is completed
+/**
+ * Refactored TPC-C Benchmark Loader for Aurora DSQL
+ *
+ * This is a cleaner, more maintainable version of the original DSQLTPCCLoader. Key improvements:
+ * - Separated concerns with dedicated manager classes - Centralized constants - Cleaner retry logic
+ * - Modular table loaders - Better error handling
*/
public final class DSQLTPCCLoader extends Loader {
- private static final int FIRST_UNPROCESSED_O_ID = 2101;
-
- private static final int PROGRESS_BAR_LENGTH = 30;
-
- private final long numWarehouses;
+ private final int startWarehouseIndex;
+ private final int endWarehouseIndex;
+ private final int stride;
- private final AtomicInteger warehousesLoaded;
+ private final ConnectionManager connectionManager;
+ private final RetryHandler retryHandler;
public DSQLTPCCLoader(TPCCBenchmark benchmark) {
super(benchmark);
- numWarehouses = Math.max(Math.round(TPCCConfig.configWhseCount * this.scaleFactor), 1);
- warehousesLoaded = new AtomicInteger(0);
- }
-
- @Override
- public List createLoaderThreads() {
- List threads = new ArrayList<>();
- final CountDownLatch itemLatch = new CountDownLatch(1);
- final CountDownLatch warehouseLatch = new CountDownLatch((int) this.numWarehouses);
-
- // ITEM
- // This will be invoked first and executed in a single thread.
- threads.add(
- new LoaderThread(this.benchmark) {
- @Override
- public void load(Connection conn) {
- loadItems(conn, TPCCConfig.configItemCount);
- }
-
- @Override
- public void afterLoad() {
- itemLatch.countDown();
- }
- });
-
- // WAREHOUSES
- // We use a separate thread per warehouse. Each thread will load
- // all of the tables that depend on that warehouse. They all have
- // to wait until the ITEM table is loaded first though.
- for (int w = 1; w <= numWarehouses; w++) {
- final int w_id = w;
- LoaderThread t =
- new LoaderThread(this.benchmark) {
- @Override
- public void load(Connection conn) {
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Starting to load WAREHOUSE {}", w_id);
- }
- // WAREHOUSE
- conn = loadWarehouse(conn, w_id);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Starting to load STOCK {}", w_id);
- }
- // STOCK
- conn = loadStock(conn, w_id, TPCCConfig.configItemCount);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Starting to load DISTRICT {}", w_id);
- }
- // DISTRICT
- conn = loadDistricts(conn, w_id, TPCCConfig.configDistPerWhse);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Starting to load CUSTOMER {}", w_id);
- }
- // CUSTOMER
- conn =
- loadCustomers(
- conn, w_id, TPCCConfig.configDistPerWhse, TPCCConfig.configCustPerDist);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Starting to load CUSTOMER HISTORY {}", w_id);
- }
- // CUSTOMER HISTORY
- conn =
- loadCustomerHistory(
- conn, w_id, TPCCConfig.configDistPerWhse, TPCCConfig.configCustPerDist);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Starting to load ORDERS {}", w_id);
- }
- // ORDERS
- conn =
- loadOpenOrders(
- conn, w_id, TPCCConfig.configDistPerWhse, TPCCConfig.configCustPerDist);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Starting to load NEW ORDERS {}", w_id);
- }
- // NEW ORDERS
- conn =
- loadNewOrders(
- conn, w_id, TPCCConfig.configDistPerWhse, TPCCConfig.configCustPerDist);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Starting to load ORDER LINES {}", w_id);
- }
- // ORDER LINES
- loadOrderLines(
- conn, w_id, TPCCConfig.configDistPerWhse, TPCCConfig.configCustPerDist);
- }
-
- @Override
- public void beforeLoad() {
-
- // Make sure that we load the ITEM table first
-
- try {
- itemLatch.await();
- } catch (InterruptedException ex) {
- throw new RuntimeException(ex);
- }
- }
- @Override
- public void afterLoad() {
- warehouseLatch.countDown();
- logProgress(warehousesLoaded.incrementAndGet(), numWarehouses);
- }
- };
- threads.add(t);
- }
-
- // POST LOAD ANALYZE
- // This will run analyze on all the Tables in TPCC.
- threads.add(
- new LoaderThread(this.benchmark) {
- @Override
- public void load(Connection conn) {
- String[] tableNames =
- new String[] {
- TPCCConstants.TABLENAME_ITEM,
- TPCCConstants.TABLENAME_WAREHOUSE,
- TPCCConstants.TABLENAME_STOCK,
- TPCCConstants.TABLENAME_DISTRICT,
- TPCCConstants.TABLENAME_CUSTOMER,
- TPCCConstants.TABLENAME_HISTORY,
- TPCCConstants.TABLENAME_OPENORDER,
- TPCCConstants.TABLENAME_NEWORDER,
- TPCCConstants.TABLENAME_ORDERLINE
- };
- LOG.info("Running ANALYZE on all tables...");
- runAnalyze(conn, tableNames);
- }
+ validateConfiguration();
- @Override
- public void beforeLoad() {
- // Make sure that we load the all the warehouses and their data first
- try {
- warehouseLatch.await();
- } catch (InterruptedException ex) {
- throw new RuntimeException(ex);
- }
- }
+ this.startWarehouseIndex = workConf.getStartWarehouseIndex();
+ this.endWarehouseIndex = workConf.getEndWarehouseIndex();
+ this.stride = workConf.getStride();
- @Override
- public void afterLoad() {
- LOG.info("ANALYZE complete!");
- }
- });
-
- return (threads);
- }
-
- private PreparedStatement getInsertStatement(Connection conn, String tableName)
- throws SQLException {
- Table catalog_tbl = benchmark.getCatalog().getTable(tableName);
- String sql = SQLUtil.getInsertSQL(catalog_tbl, this.getDatabaseType());
- return conn.prepareStatement(sql);
- }
-
- /*
- * This custom function keeps retrying the data inserts until
- * the load succeeds or the attempts reach the max retry count.
- * Since new connections are created at each retry attempt, the
- * function returns the last successful connection to the db in
- * order to allow the next loader in the same thread to re-use it.
- */
- private Connection executeInsertStatmentWithRetry(
- Connection conn, Consumer insertCallable, String tableName) {
- int attempts = 0;
- PreparedStatement stmt;
- while (attempts <= this.workConf.getMaxRetries()) {
- try {
- stmt = getInsertStatement(conn, tableName);
- insertCallable.accept(stmt);
- return conn;
- } catch (Exception e) {
- final Throwable t = e.getCause();
- if (isDuplicateKeyException(t)) {
- LOG.warn("Skipping insert due to duplicate key violation");
- return conn;
- }
-
- attempts++;
- if (attempts >= this.workConf.getMaxRetries()) {
- throw new RuntimeException("Load attempts exhausted", e);
- }
+ this.connectionManager = new ConnectionManager(benchmark);
+ this.retryHandler = new RetryHandler(workConf.getMaxRetries(), connectionManager);
- LOG.warn("[Attempt: " + attempts + "]Batch insert failed with exception, retrying...");
-
- // Wait before retrying
- try {
- // Exponential delay with jitter
- long delay = calExpDelay(attempts);
- Thread.sleep(delay);
- } catch (InterruptedException ie) {
- throw new RuntimeException("Interrupted while retrying data load ", ie);
- }
-
- // Replace old Connection with new Connection
- // And Close previous connection and prepared statement
- try {
- Connection newConnection = ConnectionUtil.makeConnectionWithRetry(this.benchmark);
-
- if (!conn.isClosed()) {
- getInsertStatement(conn, tableName).close();
- conn.close();
- }
-
- conn = newConnection;
- } catch (SQLException se) {
- throw new RuntimeException("Failed to create connection while retrying data load ", se);
- }
- }
- }
- return conn;
- }
-
- /*
- * This custom function keeps retrying simple statements until
- * the load succeeds or the attempts reach the max retry count.
- * Since new connections are created at each retry attempt, the
- * function returns the last successful connection to the db in
- * order to allow the next loader in the same thread to re-use it.
- */
- private Connection executeStatmentWithRetry(Connection conn, Consumer callable) {
- int attempts = 0;
- Statement stmt;
- while (attempts <= this.workConf.getMaxRetries()) {
- try {
- stmt = conn.createStatement();
- callable.accept(stmt);
- return conn;
- } catch (Exception e) {
- final Throwable t = e.getCause();
- if (isDuplicateKeyException(t)) {
- LOG.warn("Skipping sql execution due to duplicate key violation");
- return conn;
- }
-
- attempts++;
- if (attempts >= this.workConf.getMaxRetries()) {
- throw new RuntimeException("Execution attempts exhausted", e);
- }
-
- LOG.warn(
- "[Attempt: "
- + attempts
- + "]SQL statement execution failed with exception, retrying...");
-
- // Wait before retrying
- try {
- // Exponential delay with jitter
- long delay = calExpDelay(attempts);
- Thread.sleep(delay);
- } catch (InterruptedException ie) {
- throw new RuntimeException("Interrupted while retrying SQL statement execution", ie);
- }
-
- // Replace old Connection with new Connection
- // And Close previous connection and prepared statement
- try {
- Connection newConnection = ConnectionUtil.makeConnectionWithRetry(this.benchmark);
-
- if (!conn.isClosed()) {
- conn.close();
- }
-
- conn = newConnection;
- } catch (SQLException se) {
- throw new RuntimeException(
- "Failed to create connection while retrying SQL statement ", se);
- }
- }
- }
- return conn;
- }
-
- private static long calExpDelay(int attempts) {
- long baseDelay = 1000; // in milliseconds
- double jitterFactor = 1.0;
-
- long delay = (long) (baseDelay * Math.pow(2, attempts));
- delay = (long) (delay * (1 + jitterFactor * Math.random()));
- delay = Math.min(delay, 4000);
-
- return delay;
- }
-
- private boolean isDuplicateKeyException(Throwable t) {
- if (t.getMessage() != null && t.getMessage().contains("duplicate")) {
- return true;
- } else if (t.getCause() != null) {
- return isDuplicateKeyException(t.getCause());
- }
- return false;
- }
-
- private void logProgress(int warehouseLoaded, long numWarehouse) {
- double progess = (double) warehouseLoaded / (double) numWarehouse;
- progess = Math.max(0, Math.min(1, progess));
-
- int filled = (int) (PROGRESS_BAR_LENGTH * progess);
- StringBuilder bar = new StringBuilder();
-
- for (int i = 0; i < PROGRESS_BAR_LENGTH; i++) {
- if (i < filled) {
- bar.append("█");
- } else {
- bar.append("-");
- }
- }
LOG.info(
- String.format(
- "Load Progress:\t[%s] %d/%d %s",
- bar.toString(),
- warehouseLoaded,
- numWarehouse,
- numWarehouse > 1 ? "warehouses" : "warehouse"));
- }
-
- protected Connection loadItems(Connection conn, int itemCount) {
- List- items = new ArrayList<>();
- for (int i = 1; i <= itemCount; i++) {
-
- Item item = new Item();
- item.i_id = i;
- item.i_name = TPCCUtil.randomStr(TPCCUtil.randomNumber(14, 24, benchmark.rng()));
- item.i_price = TPCCUtil.randomNumber(100, 10000, benchmark.rng()) / 100.0;
-
- // i_data
- int randPct = TPCCUtil.randomNumber(1, 100, benchmark.rng());
- int len = TPCCUtil.randomNumber(26, 50, benchmark.rng());
- if (randPct > 10) {
- // 90% of time i_data isa random string of length [26 .. 50]
- item.i_data = TPCCUtil.randomStr(len);
- } else {
- // 10% of time i_data has "ORIGINAL" crammed somewhere in
- // middle
- int startORIGINAL = TPCCUtil.randomNumber(2, (len - 8), benchmark.rng());
- item.i_data =
- TPCCUtil.randomStr(startORIGINAL - 1)
- + "ORIGINAL"
- + TPCCUtil.randomStr(len - startORIGINAL - 9);
- }
-
- item.i_im_id = TPCCUtil.randomNumber(1, 10000, benchmark.rng());
-
- items.add(item);
-
- if (items.size() == workConf.getBatchSize()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertItems(items, stmt);
- },
- TPCCConstants.TABLENAME_ITEM);
- items.clear();
- }
- }
-
- if (!items.isEmpty()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertItems(items, stmt);
- },
- TPCCConstants.TABLENAME_ITEM);
- items.clear();
- }
- return conn;
+ "TPCC Loader Configuration: skipMainData={}, skipItems={}, skipIndex={}, "
+ + "warehouses=[{}-{}], stride={}",
+ workConf.skipMainDataLoad(),
+ workConf.skipItemLoad(),
+ workConf.skipIndexBuild(),
+ startWarehouseIndex,
+ endWarehouseIndex,
+ stride);
}
- private void insertItems(List
- items, PreparedStatement itemPrepStmt) {
- try {
- for (Item item : items) {
- int idx = 1;
- itemPrepStmt.setLong(idx++, item.i_id);
- itemPrepStmt.setString(idx++, item.i_name);
- itemPrepStmt.setDouble(idx++, item.i_price);
- itemPrepStmt.setString(idx++, item.i_data);
- itemPrepStmt.setLong(idx, item.i_im_id);
- itemPrepStmt.addBatch();
- }
- itemPrepStmt.executeBatch();
- itemPrepStmt.clearBatch();
- } catch (SQLException e) {
- throw new RuntimeException("Failed to insert items", e);
- }
+ private void validateConfiguration() {
+ Preconditions.checkArgument(
+ workConf.getStartWarehouseIndex() >= 1,
+ "Start warehouse index must be >= 1, but was: %s",
+ (Object) workConf.getStartWarehouseIndex());
+
+ Preconditions.checkArgument(
+ workConf.getEndWarehouseIndex() >= 1,
+ "End warehouse index must be >= 1, but was: %s",
+ (Object) workConf.getEndWarehouseIndex());
+
+ Preconditions.checkArgument(
+ workConf.getEndWarehouseIndex() <= workConf.getScaleFactor(),
+ "End warehouse index must be <= scale factor. End index: %s, Scale factor: %s",
+ (Object) workConf.getEndWarehouseIndex(),
+ (Object) workConf.getScaleFactor());
+
+ Preconditions.checkArgument(
+ workConf.getStride() >= 1,
+ "Stride must be >= 1, but was: %s",
+ (Object) workConf.getStride());
+
+ Preconditions.checkArgument(
+ workConf.getStartWarehouseIndex() <= workConf.getEndWarehouseIndex(),
+ "Start warehouse index must be <= end warehouse index. Start: %s, End: %s",
+ (Object) workConf.getStartWarehouseIndex(),
+ (Object) workConf.getEndWarehouseIndex());
}
- protected Connection loadWarehouse(Connection conn, int w_id) {
-
- Warehouse warehouse = new Warehouse();
+ @Override
+ public List createLoaderThreads() {
+ List threads = new ArrayList<>();
- warehouse.w_id = w_id;
- warehouse.w_ytd = 300000;
+ // Calculate number of warehouses to load
+ int numWarehouses = calculateWarehouseCount();
- // random within [0.0000 .. 0.2000]
- warehouse.w_tax = (TPCCUtil.randomNumber(0, 2000, benchmark.rng())) / 10000.0;
- warehouse.w_name = TPCCUtil.randomStr(TPCCUtil.randomNumber(6, 10, benchmark.rng()));
- warehouse.w_street_1 = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
- warehouse.w_street_2 = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
- warehouse.w_city = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
- warehouse.w_state = TPCCUtil.randomStr(3).toUpperCase();
- warehouse.w_zip = "123456789";
+ // Create latches for coordination
+ CountDownLatch itemLatch = new CountDownLatch(1);
+ CountDownLatch indexLatch = new CountDownLatch(1);
+ CountDownLatch allThreadLatch = new CountDownLatch(2 + numWarehouses);
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertWarehouse(warehouse, stmt);
- },
- TPCCConstants.TABLENAME_WAREHOUSE);
+ // Create threads
+ threads.add(createIndexThread(indexLatch, allThreadLatch));
+ threads.add(createItemLoaderThread(itemLatch, indexLatch, allThreadLatch));
+ threads.addAll(createWarehouseLoaderThreads(itemLatch, indexLatch, allThreadLatch));
- return conn;
+ return threads;
}
- private void insertWarehouse(Warehouse warehouse, PreparedStatement whsePrepStmt) {
- try {
- int idx = 1;
- whsePrepStmt.setLong(idx++, warehouse.w_id);
- whsePrepStmt.setDouble(idx++, warehouse.w_ytd);
- whsePrepStmt.setDouble(idx++, warehouse.w_tax);
- whsePrepStmt.setString(idx++, warehouse.w_name);
- whsePrepStmt.setString(idx++, warehouse.w_street_1);
- whsePrepStmt.setString(idx++, warehouse.w_street_2);
- whsePrepStmt.setString(idx++, warehouse.w_city);
- whsePrepStmt.setString(idx++, warehouse.w_state);
- whsePrepStmt.setString(idx, warehouse.w_zip);
- whsePrepStmt.execute();
- } catch (SQLException sqlException) {
- throw new RuntimeException("Failed to insert warehouse", sqlException);
+ private int calculateWarehouseCount() {
+ int count = 0;
+ for (int w = startWarehouseIndex; w <= endWarehouseIndex; w += stride) {
+ count++;
}
+ return count;
}
- protected Connection loadStock(Connection conn, int w_id, int numItems) {
-
- List stocks = new ArrayList<>();
-
- for (int i = 1; i <= numItems; i++) {
- Stock stock = new Stock();
- stock.s_i_id = i;
- stock.s_w_id = w_id;
- stock.s_quantity = TPCCUtil.randomNumber(10, 100, benchmark.rng());
- stock.s_ytd = 0;
- stock.s_order_cnt = 0;
- stock.s_remote_cnt = 0;
-
- // s_data
- int randPct = TPCCUtil.randomNumber(1, 100, benchmark.rng());
- int len = TPCCUtil.randomNumber(26, 50, benchmark.rng());
- if (randPct > 10) {
- // 90% of time i_data isa random string of length [26 ..
- // 50]
- stock.s_data = TPCCUtil.randomStr(len);
- } else {
- // 10% of time i_data has "ORIGINAL" crammed somewhere
- // in middle
- int startORIGINAL = TPCCUtil.randomNumber(2, (len - 8), benchmark.rng());
- stock.s_data =
- TPCCUtil.randomStr(startORIGINAL - 1)
- + "ORIGINAL"
- + TPCCUtil.randomStr(len - startORIGINAL - 9);
- }
- stocks.add(stock);
-
- if (stocks.size() == workConf.getBatchSize()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertStock(stocks, stmt);
- },
- TPCCConstants.TABLENAME_STOCK);
- stocks.clear();
- }
+ private LoaderThread createIndexThread(CountDownLatch indexLatch, CountDownLatch allThreadLatch) {
+ if (workConf.skipIndexBuild() || !DatabaseType.AURORADSQL.equals(workConf.getDatabaseType())) {
+ LOG.info("Skipping index creation");
+ indexLatch.countDown();
+ allThreadLatch.countDown();
+ return new NoOpLoaderThread(this.benchmark);
}
-
- if (!stocks.isEmpty()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertStock(stocks, stmt);
- },
- TPCCConstants.TABLENAME_STOCK);
- stocks.clear();
- }
- return conn;
- }
-
- private void insertStock(List stocks, PreparedStatement stockPreparedStatement) {
- try {
- for (Stock stock : stocks) {
- int idx = 1;
- stockPreparedStatement.setLong(idx++, stock.s_w_id);
- stockPreparedStatement.setLong(idx++, stock.s_i_id);
- stockPreparedStatement.setLong(idx++, stock.s_quantity);
- stockPreparedStatement.setDouble(idx++, stock.s_ytd);
- stockPreparedStatement.setLong(idx++, stock.s_order_cnt);
- stockPreparedStatement.setLong(idx++, stock.s_remote_cnt);
- stockPreparedStatement.setString(idx++, stock.s_data);
- stockPreparedStatement.setString(idx++, TPCCUtil.randomStr(24));
- stockPreparedStatement.setString(idx++, TPCCUtil.randomStr(24));
- stockPreparedStatement.setString(idx++, TPCCUtil.randomStr(24));
- stockPreparedStatement.setString(idx++, TPCCUtil.randomStr(24));
- stockPreparedStatement.setString(idx++, TPCCUtil.randomStr(24));
- stockPreparedStatement.setString(idx++, TPCCUtil.randomStr(24));
- stockPreparedStatement.setString(idx++, TPCCUtil.randomStr(24));
- stockPreparedStatement.setString(idx++, TPCCUtil.randomStr(24));
- stockPreparedStatement.setString(idx++, TPCCUtil.randomStr(24));
- stockPreparedStatement.setString(idx, TPCCUtil.randomStr(24));
- stockPreparedStatement.addBatch();
+ return new LoaderThread(this.benchmark) {
+ @Override
+ public void load(Connection conn) {
+ try {
+ createIndexAsync(conn);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to create index", e);
+ }
}
- stockPreparedStatement.executeBatch();
- stockPreparedStatement.clearBatch();
- } catch (SQLException sqlException) {
- throw new RuntimeException("Failed to insert stocks", sqlException);
- }
- }
-
- protected Connection loadDistricts(Connection conn, int w_id, int districtsPerWarehouse) {
- for (int d = 1; d <= districtsPerWarehouse; d++) {
- District district = new District();
- district.d_id = d;
- district.d_w_id = w_id;
- district.d_ytd = 30000;
-
- // random within [0.0000 .. 0.2000]
- district.d_tax = (float) ((TPCCUtil.randomNumber(0, 2000, benchmark.rng())) / 10000.0);
-
- district.d_next_o_id = TPCCConfig.configCustPerDist + 1;
- district.d_name = TPCCUtil.randomStr(TPCCUtil.randomNumber(6, 10, benchmark.rng()));
- district.d_street_1 = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
- district.d_street_2 = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
- district.d_city = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
- district.d_state = TPCCUtil.randomStr(3).toUpperCase();
- district.d_zip = "123456789";
-
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertDistrict(district, stmt);
- },
- TPCCConstants.TABLENAME_DISTRICT);
- }
- return conn;
+ @Override
+ public void afterLoad() {
+ indexLatch.countDown();
+ allThreadLatch.countDown();
+ }
+ };
}
- private void insertDistrict(District district, PreparedStatement distPrepStmt) {
- try {
- int idx = 1;
- distPrepStmt.setLong(idx++, district.d_w_id);
- distPrepStmt.setLong(idx++, district.d_id);
- distPrepStmt.setDouble(idx++, district.d_ytd);
- distPrepStmt.setDouble(idx++, district.d_tax);
- distPrepStmt.setLong(idx++, district.d_next_o_id);
- distPrepStmt.setString(idx++, district.d_name);
- distPrepStmt.setString(idx++, district.d_street_1);
- distPrepStmt.setString(idx++, district.d_street_2);
- distPrepStmt.setString(idx++, district.d_city);
- distPrepStmt.setString(idx++, district.d_state);
- distPrepStmt.setString(idx, district.d_zip);
- distPrepStmt.executeUpdate();
- } catch (SQLException sqlException) {
- throw new RuntimeException("Failed to insert districts", sqlException);
+ private LoaderThread createItemLoaderThread(
+ CountDownLatch itemLatch, CountDownLatch indexLatch, CountDownLatch allThreadLatch) {
+ if (workConf.skipItemLoad()) {
+ LOG.info("Skipping item load");
+ itemLatch.countDown();
+ allThreadLatch.countDown();
+ return new NoOpLoaderThread(this.benchmark);
}
- }
- protected Connection loadCustomers(
- Connection conn, int w_id, int districtsPerWarehouse, int customersPerDistrict) {
-
- List customers = new ArrayList<>();
-
- for (int d = 1; d <= districtsPerWarehouse; d++) {
- for (int c = 1; c <= customersPerDistrict; c++) {
- Timestamp sysdate = new Timestamp(System.currentTimeMillis());
+ return new LoaderThread(this.benchmark) {
+ @Override
+ public void load(Connection conn) {
+ String threadName = TPCCLoaderConstants.LOAD_ITEMS_THREAD_NAME;
+ try {
+ connectionManager.createConnection(threadName);
- Customer customer = new Customer();
- customer.c_id = c;
- customer.c_d_id = d;
- customer.c_w_id = w_id;
+ ItemTableLoader itemLoader =
+ new ItemTableLoader(
+ benchmark, connectionManager, retryHandler, workConf.getBatchSize());
- // discount is random between [0.0000 ... 0.5000]
- customer.c_discount = (float) (TPCCUtil.randomNumber(1, 5000, benchmark.rng()) / 10000.0);
+ itemLoader.load(threadName);
- if (TPCCUtil.randomNumber(1, 100, benchmark.rng()) <= 10) {
- customer.c_credit = "BC"; // 10% Bad Credit
- } else {
- customer.c_credit = "GC"; // 90% Good Credit
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to load items", e);
}
- if (c <= 1000) {
- customer.c_last = TPCCUtil.getLastName(c - 1);
- } else {
- customer.c_last = TPCCUtil.getNonUniformRandomLastNameForLoad(benchmark.rng());
- }
- customer.c_first = TPCCUtil.randomStr(TPCCUtil.randomNumber(8, 16, benchmark.rng()));
- customer.c_credit_lim = 50000;
-
- customer.c_balance = -10;
- customer.c_ytd_payment = 10;
- customer.c_payment_cnt = 1;
- customer.c_delivery_cnt = 0;
-
- customer.c_street_1 = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
- customer.c_street_2 = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
- customer.c_city = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
- customer.c_state = TPCCUtil.randomStr(3).toUpperCase();
- // TPC-C 4.3.2.7: 4 random digits + "11111"
- customer.c_zip = TPCCUtil.randomNStr(4) + "11111";
- customer.c_phone = TPCCUtil.randomNStr(16);
- customer.c_since = sysdate;
- customer.c_middle = "OE";
- customer.c_data = TPCCUtil.randomStr(TPCCUtil.randomNumber(300, 500, benchmark.rng()));
-
- customers.add(customer);
+ }
- if (customers.size() == workConf.getBatchSize()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertCustomer(customers, stmt);
- },
- TPCCConstants.TABLENAME_CUSTOMER);
- customers.clear();
+ @Override
+ public void beforeLoad() {
+ try {
+ indexLatch.await();
+ Thread.sleep(TPCCLoaderConstants.POST_INDEX_WAIT_MS);
+ } catch (InterruptedException e) {
+ throw new RuntimeException("Interrupted while waiting for index creation", e);
}
}
- }
- if (!customers.isEmpty()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertCustomer(customers, stmt);
- },
- TPCCConstants.TABLENAME_CUSTOMER);
- customers.clear();
- }
-
- return conn;
- }
-
- private void insertCustomer(List customers, PreparedStatement custPrepStmt) {
- try {
- for (Customer customer : customers) {
- int idx = 1;
- custPrepStmt.setLong(idx++, customer.c_w_id);
- custPrepStmt.setLong(idx++, customer.c_d_id);
- custPrepStmt.setLong(idx++, customer.c_id);
- custPrepStmt.setDouble(idx++, customer.c_discount);
- custPrepStmt.setString(idx++, customer.c_credit);
- custPrepStmt.setString(idx++, customer.c_last);
- custPrepStmt.setString(idx++, customer.c_first);
- custPrepStmt.setDouble(idx++, customer.c_credit_lim);
- custPrepStmt.setDouble(idx++, customer.c_balance);
- custPrepStmt.setDouble(idx++, customer.c_ytd_payment);
- custPrepStmt.setLong(idx++, customer.c_payment_cnt);
- custPrepStmt.setLong(idx++, customer.c_delivery_cnt);
- custPrepStmt.setString(idx++, customer.c_street_1);
- custPrepStmt.setString(idx++, customer.c_street_2);
- custPrepStmt.setString(idx++, customer.c_city);
- custPrepStmt.setString(idx++, customer.c_state);
- custPrepStmt.setString(idx++, customer.c_zip);
- custPrepStmt.setString(idx++, customer.c_phone);
- custPrepStmt.setTimestamp(idx++, customer.c_since);
- custPrepStmt.setString(idx++, customer.c_middle);
- custPrepStmt.setString(idx, customer.c_data);
- custPrepStmt.addBatch();
+ @Override
+ public void afterLoad() {
+ itemLatch.countDown();
+ allThreadLatch.countDown();
+ connectionManager.closeResourcesForThread(TPCCLoaderConstants.LOAD_ITEMS_THREAD_NAME);
}
-
- custPrepStmt.executeBatch();
- custPrepStmt.clearBatch();
- } catch (SQLException sqlException) {
- throw new RuntimeException("Failed to insert customers", sqlException);
- }
+ };
}
- protected Connection loadCustomerHistory(
- Connection conn, int w_id, int districtsPerWarehouse, int customersPerDistrict) {
-
- List historyList = new ArrayList<>();
-
- for (int d = 1; d <= districtsPerWarehouse; d++) {
- for (int c = 1; c <= customersPerDistrict; c++) {
- Timestamp sysdate = new Timestamp(System.currentTimeMillis());
-
- History history = new History();
- history.h_c_id = c;
- history.h_c_d_id = d;
- history.h_c_w_id = w_id;
- history.h_d_id = d;
- history.h_w_id = w_id;
- history.h_date = sysdate;
- history.h_amount = 10;
- history.h_data = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 24, benchmark.rng()));
-
- historyList.add(history);
+ private List createWarehouseLoaderThreads(
+ CountDownLatch itemLatch, CountDownLatch indexLatch, CountDownLatch allThreadLatch) {
+ List threads = new ArrayList<>();
- if (historyList.size() == workConf.getBatchSize()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertCustomerHistory(historyList, stmt);
- },
- TPCCConstants.TABLENAME_HISTORY);
- historyList.clear();
- }
+ if (workConf.skipMainDataLoad()) {
+ LOG.info("Skipping main data load");
+ int numWarehouses = calculateWarehouseCount();
+ for (int i = 0; i < numWarehouses; i++) {
+ allThreadLatch.countDown();
}
+ return threads;
}
- if (!historyList.isEmpty()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertCustomerHistory(historyList, stmt);
- },
- TPCCConstants.TABLENAME_HISTORY);
- historyList.clear();
+ for (int w = startWarehouseIndex; w <= endWarehouseIndex; w += stride) {
+ final int warehouseId = w;
+ threads.add(createWarehouseThread(warehouseId, itemLatch, indexLatch, allThreadLatch));
}
- return conn;
+ return threads;
}
- private void insertCustomerHistory(List historyList, PreparedStatement histPrepStmt) {
- try {
- for (History history : historyList) {
- int idx = 1;
- histPrepStmt.setInt(idx++, history.h_c_id);
- histPrepStmt.setInt(idx++, history.h_c_d_id);
- histPrepStmt.setInt(idx++, history.h_c_w_id);
- histPrepStmt.setInt(idx++, history.h_d_id);
- histPrepStmt.setInt(idx++, history.h_w_id);
- histPrepStmt.setTimestamp(idx++, history.h_date);
- histPrepStmt.setDouble(idx++, history.h_amount);
- histPrepStmt.setString(idx, history.h_data);
- histPrepStmt.addBatch();
- }
-
- histPrepStmt.executeBatch();
- histPrepStmt.clearBatch();
- } catch (SQLException sqlException) {
- throw new RuntimeException("Failed to insert history", sqlException);
- }
- }
-
- protected Connection loadOpenOrders(
- Connection conn, int w_id, int districtsPerWarehouse, int customersPerDistrict) {
-
- List oorders = new ArrayList<>();
+ private LoaderThread createWarehouseThread(
+ int warehouseId,
+ CountDownLatch itemLatch,
+ CountDownLatch indexLatch,
+ CountDownLatch allThreadLatch) {
+ return new LoaderThread(this.benchmark) {
+ private final String threadName = String.valueOf(warehouseId);
- for (int d = 1; d <= districtsPerWarehouse; d++) {
- // TPC-C 4.3.3.1: o_c_id must be a permutation of [1, 3000]
- int[] c_ids = new int[customersPerDistrict];
- for (int i = 0; i < customersPerDistrict; ++i) {
- c_ids[i] = i + 1;
- }
- // Collections.shuffle exists, but there is no
- // Arrays.shuffle
- for (int i = 0; i < c_ids.length - 1; ++i) {
- int remaining = c_ids.length - i - 1;
- int swapIndex = benchmark.rng().nextInt(remaining) + i + 1;
-
- int temp = c_ids[swapIndex];
- c_ids[swapIndex] = c_ids[i];
- c_ids[i] = temp;
- }
-
- for (int c = 1; c <= customersPerDistrict; c++) {
-
- Oorder oorder = new Oorder();
- oorder.o_id = c;
- oorder.o_w_id = w_id;
- oorder.o_d_id = d;
- oorder.o_c_id = c_ids[c - 1];
- // o_carrier_id is set *only* for orders with ids < 2101
- // [4.3.3.1]
- if (oorder.o_id < FIRST_UNPROCESSED_O_ID) {
- oorder.o_carrier_id = TPCCUtil.randomNumber(1, 10, benchmark.rng());
- } else {
- oorder.o_carrier_id = null;
- }
- oorder.o_ol_cnt = getRandomCount(w_id, c, d);
- oorder.o_all_local = 1;
- oorder.o_entry_d = new Timestamp(System.currentTimeMillis());
-
- oorders.add(oorder);
-
- if (oorders.size() == workConf.getBatchSize()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertOpenOrders(oorders, stmt);
- },
- TPCCConstants.TABLENAME_OPENORDER);
- oorders.clear();
+ @Override
+ public void beforeLoad() {
+ try {
+ indexLatch.await();
+ itemLatch.await();
+ } catch (InterruptedException e) {
+ throw new RuntimeException("Interrupted while waiting", e);
}
}
- }
- if (!oorders.isEmpty()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertOpenOrders(oorders, stmt);
- },
- TPCCConstants.TABLENAME_OPENORDER);
- oorders.clear();
- }
-
- return conn;
- }
-
- private void insertOpenOrders(List oorders, PreparedStatement openOrderStatement) {
- try {
- for (Oorder oorder : oorders) {
- int idx = 1;
- openOrderStatement.setInt(idx++, oorder.o_w_id);
- openOrderStatement.setInt(idx++, oorder.o_d_id);
- openOrderStatement.setInt(idx++, oorder.o_id);
- openOrderStatement.setInt(idx++, oorder.o_c_id);
- if (oorder.o_carrier_id != null) {
- openOrderStatement.setInt(idx++, oorder.o_carrier_id);
- } else {
- openOrderStatement.setNull(idx++, Types.INTEGER);
+ @Override
+ public void load(Connection conn) {
+ try {
+ connectionManager.createConnection(threadName);
+ loadWarehouseData(warehouseId);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to load warehouse " + warehouseId, e);
}
- openOrderStatement.setInt(idx++, oorder.o_ol_cnt);
- openOrderStatement.setInt(idx++, oorder.o_all_local);
- openOrderStatement.setTimestamp(idx, oorder.o_entry_d);
- openOrderStatement.addBatch();
}
- openOrderStatement.executeBatch();
- openOrderStatement.clearBatch();
- } catch (SQLException sqlException) {
- throw new RuntimeException("Failed to insert open orders", sqlException);
- }
- }
-
- private int getRandomCount(int w_id, int c, int d) {
- Customer customer = new Customer();
- customer.c_id = c;
- customer.c_d_id = d;
- customer.c_w_id = w_id;
-
- Random random = new Random(customer.hashCode());
-
- return TPCCUtil.randomNumber(5, 15, random);
- }
-
- protected Connection loadNewOrders(
- Connection conn, int w_id, int districtsPerWarehouse, int customersPerDistrict) {
-
- List newOrders = new ArrayList<>();
-
- for (int d = 1; d <= districtsPerWarehouse; d++) {
-
- for (int c = 1; c <= customersPerDistrict; c++) {
-
- // 900 rows in the NEW-ORDER table corresponding to the last
- // 900 rows in the ORDER table for that district (i.e.,
- // with NO_O_ID between 2,101 and 3,000)
- if (c >= FIRST_UNPROCESSED_O_ID) {
- NewOrder new_order = new NewOrder();
- new_order.no_w_id = w_id;
- new_order.no_d_id = d;
- new_order.no_o_id = c;
+ @Override
+ public void afterLoad() {
+ allThreadLatch.countDown();
+ connectionManager.closeResourcesForThread(threadName);
+ }
- newOrders.add(new_order);
- }
+ private void loadWarehouseData(int warehouseId) throws SQLException {
+
+ LOG.info("Loading all data for warehouse {}", warehouseId);
+
+ // Create loader instances
+ WarehouseTableLoader warehouseLoader =
+ new WarehouseTableLoader(
+ benchmark, connectionManager, retryHandler, workConf.getBatchSize());
+ StockTableLoader stockLoader =
+ new StockTableLoader(
+ benchmark, connectionManager, retryHandler, workConf.getBatchSize());
+ DistrictTableLoader districtLoader =
+ new DistrictTableLoader(
+ benchmark, connectionManager, retryHandler, workConf.getBatchSize());
+ CustomerTableLoader customerLoader =
+ new CustomerTableLoader(
+ benchmark, connectionManager, retryHandler, workConf.getBatchSize());
+ HistoryTableLoader historyLoader =
+ new HistoryTableLoader(
+ benchmark, connectionManager, retryHandler, workConf.getBatchSize());
+ OrderTableLoader orderLoader =
+ new OrderTableLoader(
+ benchmark, connectionManager, retryHandler, workConf.getBatchSize());
+ NewOrderTableLoader newOrderLoader =
+ new NewOrderTableLoader(
+ benchmark, connectionManager, retryHandler, workConf.getBatchSize());
+ OrderLineTableLoader orderLineLoader =
+ new OrderLineTableLoader(
+ benchmark, connectionManager, retryHandler, workConf.getBatchSize());
- if (newOrders.size() == workConf.getBatchSize()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertNewOrder(newOrders, stmt);
- },
- TPCCConstants.TABLENAME_NEWORDER);
- newOrders.clear();
+ try {
+ // Load in dependency order
+ warehouseLoader.load(threadName, warehouseId);
+ stockLoader.load(threadName, warehouseId);
+ districtLoader.load(threadName, warehouseId);
+ customerLoader.load(threadName, warehouseId);
+ historyLoader.load(threadName, warehouseId);
+ orderLoader.load(threadName, warehouseId);
+ newOrderLoader.load(threadName, warehouseId);
+ orderLineLoader.load(threadName, warehouseId);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to load data for warehouse " + warehouseId, e);
}
}
- }
-
- if (!newOrders.isEmpty()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertNewOrder(newOrders, stmt);
- },
- TPCCConstants.TABLENAME_NEWORDER);
- newOrders.clear();
- }
+ };
+ }
- return conn;
+ private void createIndexAsync(Connection conn) throws SQLException {
+ String jobId = submitIndexCreation(conn);
+ waitForIndexCompletion(conn, jobId);
}
- private void insertNewOrder(List newOrders, PreparedStatement newOrderStatement) {
- try {
- for (NewOrder newOrder : newOrders) {
- int idx = 1;
- newOrderStatement.setInt(idx++, newOrder.no_w_id);
- newOrderStatement.setInt(idx++, newOrder.no_d_id);
- newOrderStatement.setInt(idx, newOrder.no_o_id);
- newOrderStatement.addBatch();
+ private String submitIndexCreation(Connection conn) throws SQLException {
+ try (PreparedStatement stmt =
+ conn.prepareStatement(TPCCLoaderConstants.CREATE_CUSTOMER_INDEX_ASYNC);
+ ResultSet rs = stmt.executeQuery()) {
+ if (rs.next()) {
+ String jobId = rs.getString(1);
+ LOG.info("Index creation job started with job_id: {}", jobId);
+ return jobId;
+ } else {
+ throw new SQLException("Index creation request didn't return job id");
}
-
- newOrderStatement.executeBatch();
- newOrderStatement.clearBatch();
- } catch (SQLException sqlException) {
- throw new RuntimeException("Failed to insert new orders", sqlException);
}
}
- protected Connection loadOrderLines(
- Connection conn, int w_id, int districtsPerWarehouse, int customersPerDistrict) {
-
- List orderLines = new ArrayList<>();
-
- for (int d = 1; d <= districtsPerWarehouse; d++) {
+ private void waitForIndexCompletion(Connection conn, String jobId) throws SQLException {
+ String status = TPCCLoaderConstants.JOB_STATUS_PROCESSING;
- for (int c = 1; c <= customersPerDistrict; c++) {
+ try (PreparedStatement stmt = conn.prepareStatement(TPCCLoaderConstants.SELECT_JOB_STATUS)) {
+ stmt.setString(1, jobId);
- int count = getRandomCount(w_id, c, d);
-
- for (int l = 1; l <= count; l++) {
- OrderLine order_line = new OrderLine();
- order_line.ol_w_id = w_id;
- order_line.ol_d_id = d;
- order_line.ol_o_id = c;
- order_line.ol_number = l; // ol_number
- order_line.ol_i_id =
- TPCCUtil.randomNumber(1, TPCCConfig.configItemCount, benchmark.rng());
- if (order_line.ol_o_id < FIRST_UNPROCESSED_O_ID) {
- order_line.ol_delivery_d = new Timestamp(System.currentTimeMillis());
- order_line.ol_amount = 0;
- } else {
- order_line.ol_delivery_d = null;
- // random within [0.01 .. 9,999.99]
- order_line.ol_amount =
- (float) (TPCCUtil.randomNumber(1, 999999, benchmark.rng()) / 100.0);
+ while (!TPCCLoaderConstants.JOB_STATUS_COMPLETED.equalsIgnoreCase(status)) {
+ try (ResultSet rs = stmt.executeQuery()) {
+ if (rs.next()) {
+ status = rs.getString("status");
+ LOG.info("Index creation job {} status: {}", jobId, status);
}
- order_line.ol_supply_w_id = order_line.ol_w_id;
- order_line.ol_quantity = 5;
- order_line.ol_dist_info = TPCCUtil.randomStr(24);
-
- orderLines.add(order_line);
- if (orderLines.size() == workConf.getBatchSize()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertOrderLine(orderLines, stmt);
- },
- TPCCConstants.TABLENAME_ORDERLINE);
- orderLines.clear();
+ if (!TPCCLoaderConstants.JOB_STATUS_COMPLETED.equalsIgnoreCase(status)) {
+ Thread.sleep(TPCCLoaderConstants.INDEX_CHECK_INTERVAL_MS);
}
+ } catch (InterruptedException e) {
+ throw new SQLException("Interrupted while waiting for index creation", e);
}
}
}
-
- if (!orderLines.isEmpty()) {
- conn =
- executeInsertStatmentWithRetry(
- conn,
- (stmt) -> {
- insertOrderLine(orderLines, stmt);
- },
- TPCCConstants.TABLENAME_ORDERLINE);
- orderLines.clear();
- }
-
- return conn;
- }
-
- private void insertOrderLine(List orderLines, PreparedStatement orderLineStatement) {
- try {
- for (OrderLine orderLine : orderLines) {
- int idx = 1;
- orderLineStatement.setInt(idx++, orderLine.ol_w_id);
- orderLineStatement.setInt(idx++, orderLine.ol_d_id);
- orderLineStatement.setInt(idx++, orderLine.ol_o_id);
- orderLineStatement.setInt(idx++, orderLine.ol_number);
- orderLineStatement.setLong(idx++, orderLine.ol_i_id);
- if (orderLine.ol_delivery_d != null) {
- orderLineStatement.setTimestamp(idx++, orderLine.ol_delivery_d);
- } else {
- orderLineStatement.setNull(idx++, 0);
- }
- orderLineStatement.setDouble(idx++, orderLine.ol_amount);
- orderLineStatement.setLong(idx++, orderLine.ol_supply_w_id);
- orderLineStatement.setDouble(idx++, orderLine.ol_quantity);
- orderLineStatement.setString(idx, orderLine.ol_dist_info);
- orderLineStatement.addBatch();
- }
-
- orderLineStatement.executeBatch();
- orderLineStatement.clearBatch();
- } catch (SQLException sqlException) {
- throw new RuntimeException("Failed to insert orderline", sqlException);
- }
}
- private Connection runAnalyze(Connection conn, String[] tableNames) {
- for (String tableName : tableNames) {
- conn =
- executeStatmentWithRetry(
- conn,
- (stmt) -> {
- analyzeTables(stmt, tableName);
- });
+ /** No-op loader thread for skipped operations */
+ private static class NoOpLoaderThread extends LoaderThread {
+ public NoOpLoaderThread(TPCCBenchmark benchmark) {
+ super(benchmark);
}
- return conn;
- }
- private void analyzeTables(Statement stmt, String tableName) {
- try {
- stmt.execute("ANALYZE " + tableName);
- } catch (SQLException e) {
- throw new RuntimeException("Failed to run ANALYZE on table: " + tableName);
+ @Override
+ public void load(Connection conn) {
+ // No-op
}
}
}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/RetryHandler.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/RetryHandler.java
new file mode 100644
index 0000000..1b16947
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/RetryHandler.java
@@ -0,0 +1,193 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql;
+
+import com.oltpbenchmark.util.TimeUtil;
+import java.sql.SQLException;
+import java.util.function.Supplier;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Handles retry logic for database operations with exponential backoff. Provides a clean
+ * abstraction for retrying operations that may fail due to transient errors.
+ */
+@Slf4j
+@RequiredArgsConstructor
+public class RetryHandler {
+
+ private final int maxRetries;
+ private final ConnectionManager connectionManager;
+
+ /**
+ * Executes an operation with retry logic.
+ *
+ * @param operation The operation to execute
+ * @param threadName The thread name for connection management
+ * @param operationName Name of the operation for logging
+ * @param Return type of the operation
+ * @return The result of the operation
+ * @throws RuntimeException if all retry attempts are exhausted
+ */
+ public T executeWithRetry(Supplier operation, String threadName, String operationName) {
+ int attempts = 0;
+
+ while (attempts <= maxRetries) {
+ try {
+ return operation.get();
+ } catch (Exception e) {
+ attempts++;
+
+ if (isDuplicateKeyException(e)) {
+ log.warn("Skipping {} due to duplicate key violation", operationName);
+ return null;
+ }
+
+ if (attempts > maxRetries) {
+ throw new RuntimeException(
+ String.format("All retry attempts exhausted for %s", operationName), e);
+ }
+
+ log.error(
+ "Operation {} failed (attempt {}/{}), retrying...",
+ operationName,
+ attempts,
+ maxRetries,
+ e);
+
+ handleRetryDelay(attempts);
+ handleConnectionRecovery(threadName, e);
+ }
+ }
+
+ throw new RuntimeException(
+ String.format("Failed to execute %s after %d attempts", operationName, maxRetries));
+ }
+
+ /**
+ * Executes a database operation with retry logic specifically for SQL operations.
+ *
+ * @param operation The SQL operation to execute
+ * @param threadName The thread name for connection management
+ * @param operationName Name of the operation for logging
+ * @throws SQLException if the operation fails after all retries
+ */
+ public void executeSQLWithRetry(SQLOperation operation, String threadName, String operationName)
+ throws SQLException {
+ int attempts = 0;
+ SQLException lastException = null;
+
+ while (attempts <= maxRetries) {
+ try {
+ operation.execute();
+ return;
+ } catch (SQLException e) {
+ attempts++;
+ lastException = e;
+
+ if (isDuplicateKeyException(e)) {
+ log.warn("Skipping {} due to duplicate key violation", operationName);
+ return;
+ }
+
+ if (attempts > maxRetries) {
+ break;
+ }
+
+ log.error(
+ "SQL operation {} failed (attempt {}/{}), retrying...",
+ operationName,
+ attempts,
+ maxRetries,
+ e);
+
+ handleRetryDelay(attempts);
+ handleConnectionRecovery(threadName, e);
+ }
+ }
+
+ throw new SQLException(
+ String.format("Failed to execute %s after %d attempts", operationName, maxRetries),
+ lastException);
+ }
+
+ /** Checks if an exception is due to a duplicate key violation. */
+ private boolean isDuplicateKeyException(Throwable t) {
+ if (t == null) {
+ return false;
+ }
+
+ String message = t.getMessage();
+ if (message != null && message.toLowerCase().contains("duplicate")) {
+ return true;
+ }
+
+ return isDuplicateKeyException(t.getCause());
+ }
+
+ /** Handles the retry delay with exponential backoff. */
+ private void handleRetryDelay(int attempt) {
+ try {
+ long delay = TimeUtil.calExpDelay(attempt);
+ Thread.sleep(delay);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while waiting for retry", ie);
+ }
+ }
+
+ /** Handles connection recovery after a failure. */
+ private void handleConnectionRecovery(String threadName, Exception e) {
+ try {
+ // Check if this is a connection-related error
+ if (isConnectionError(e)) {
+ log.info("Refreshing connection for thread {} due to connection error", threadName);
+ connectionManager.refreshConnection(threadName);
+ }
+ } catch (SQLException refreshException) {
+ log.error("Failed to refresh connection for thread {}", threadName, refreshException);
+ }
+ }
+
+ /** Checks if an exception indicates a connection error. */
+ private boolean isConnectionError(Exception e) {
+ if (e instanceof SQLException) {
+ String sqlState = ((SQLException) e).getSQLState();
+ // Common SQL states for connection errors
+ return sqlState != null
+ && (sqlState.startsWith("08")
+ || // Connection exception
+ sqlState.equals("HY000")
+ || // General error (often connection-related)
+ sqlState.equals("57P01") // Admin shutdown
+ );
+ }
+
+ String message = e.getMessage();
+ return message != null
+ && (message.toLowerCase().contains("connection")
+ || message.toLowerCase().contains("closed")
+ || message.toLowerCase().contains("timeout"));
+ }
+
+ /** Functional interface for SQL operations that can throw SQLException. */
+ @FunctionalInterface
+ public interface SQLOperation {
+ void execute() throws SQLException;
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/TPCCLoaderConstants.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/TPCCLoaderConstants.java
new file mode 100644
index 0000000..5b3a350
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/TPCCLoaderConstants.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql;
+
+/**
+ * Constants used by the DSQL TPCC Loader. Centralizes all constants to improve maintainability and
+ * readability.
+ */
+public final class TPCCLoaderConstants {
+
+ private TPCCLoaderConstants() {
+ // Private constructor to prevent instantiation
+ }
+
+ // Transaction and Thread Names
+ public static final String TX_NAME = "Loader";
+ public static final String LOAD_ITEMS_THREAD_NAME = "loadItems";
+
+ // Business Logic Constants
+ public static final int FIRST_UNPROCESSED_O_ID = 2101;
+
+ // SQL Queries
+ public static final String CREATE_CUSTOMER_INDEX_ASYNC =
+ "CREATE INDEX ASYNC idx_customer_name ON customer (c_w_id, c_d_id, c_last, c_first)";
+
+ public static final String SELECT_JOB_STATUS = "SELECT status FROM sys.jobs WHERE job_id = ?";
+
+ // Index Job Status
+ public static final String JOB_STATUS_COMPLETED = "completed";
+ public static final String JOB_STATUS_PROCESSING = "processing";
+
+ // Thread Sleep Durations
+ public static final long INDEX_CHECK_INTERVAL_MS = 1000;
+ public static final long POST_INDEX_WAIT_MS = 1000;
+
+ // String Constants for Data Generation
+ public static final String ORIGINAL_STRING = "ORIGINAL";
+ public static final String BAD_CREDIT = "BC";
+ public static final String GOOD_CREDIT = "GC";
+ public static final String MIDDLE_NAME = "OE";
+ public static final String ZIP_SUFFIX = "11111";
+
+ // Data Generation Limits
+ public static final int ITEM_NAME_MIN_LENGTH = 14;
+ public static final int ITEM_NAME_MAX_LENGTH = 24;
+ public static final int ITEM_PRICE_MIN = 100;
+ public static final int ITEM_PRICE_MAX = 10000;
+ public static final int ITEM_DATA_MIN_LENGTH = 26;
+ public static final int ITEM_DATA_MAX_LENGTH = 50;
+ public static final int ORIGINAL_DATA_THRESHOLD = 10; // 10% chance
+ public static final int BAD_CREDIT_THRESHOLD = 10; // 10% chance
+
+ // Warehouse Constants
+ public static final double WAREHOUSE_INITIAL_YTD = 300000;
+ public static final int WAREHOUSE_TAX_MAX = 2000;
+
+ // District Constants
+ public static final double DISTRICT_INITIAL_YTD = 30000;
+ public static final int DISTRICT_TAX_MAX = 2000;
+
+ // Customer Constants
+ public static final double CUSTOMER_CREDIT_LIMIT = 50000;
+ public static final double CUSTOMER_INITIAL_BALANCE = -10;
+ public static final double CUSTOMER_INITIAL_YTD_PAYMENT = 10;
+ public static final int CUSTOMER_INITIAL_PAYMENT_CNT = 1;
+ public static final int CUSTOMER_INITIAL_DELIVERY_CNT = 0;
+ public static final int CUSTOMER_DISCOUNT_MAX = 5000;
+
+ // Stock Constants
+ public static final int STOCK_QUANTITY_MIN = 10;
+ public static final int STOCK_QUANTITY_MAX = 100;
+
+ // Order Constants
+ public static final int ORDER_LINE_COUNT_MIN = 5;
+ public static final int ORDER_LINE_COUNT_MAX = 15;
+ public static final int ORDER_LINE_QUANTITY = 5;
+ public static final int CARRIER_ID_MIN = 1;
+ public static final int CARRIER_ID_MAX = 10;
+
+ // History Constants
+ public static final double HISTORY_AMOUNT = 10;
+ public static final int HISTORY_DATA_MIN_LENGTH = 10;
+ public static final int HISTORY_DATA_MAX_LENGTH = 24;
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/AbstractTableLoader.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/AbstractTableLoader.java
new file mode 100644
index 0000000..22bfed5
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/AbstractTableLoader.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders;
+
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.ConnectionManager;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.RetryHandler;
+import com.oltpbenchmark.catalog.Table;
+import com.oltpbenchmark.types.DatabaseType;
+import com.oltpbenchmark.util.SQLUtil;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Abstract base class for all table loaders. Provides common functionality for loading data into
+ * TPCC tables.
+ */
+@Slf4j
+@RequiredArgsConstructor
+public abstract class AbstractTableLoader {
+
+ protected final BenchmarkModule benchmark;
+ protected final ConnectionManager connectionManager;
+ protected final RetryHandler retryHandler;
+ protected final int batchSize;
+
+ /** Gets the table name for this loader. */
+ protected abstract String getTableName();
+
+ /** Loads data for this table. */
+ public abstract void load(String threadName) throws SQLException;
+
+ /** Gets or creates a prepared statement for inserting into this table. */
+ protected PreparedStatement getInsertStatement(String threadName) throws SQLException {
+ Table catalogTable = benchmark.getCatalog().getTable(getTableName());
+ String sql = SQLUtil.getInsertSQL(catalogTable, getDatabaseType());
+ return connectionManager.getPreparedStatement(threadName, getTableName(), sql);
+ }
+
+ /** Gets the database type from the benchmark. */
+ protected DatabaseType getDatabaseType() {
+ return benchmark.getWorkloadConfiguration().getDatabaseType();
+ }
+
+ /** Executes an operation with retry logic. */
+ protected void executeWithRetry(
+ RetryHandler.SQLOperation operation, String threadName, String operationName)
+ throws SQLException {
+ retryHandler.executeSQLWithRetry(operation, threadName, operationName);
+ }
+
+ /** Cleans up resources for this loader. */
+ public void cleanup(String threadName) {
+ connectionManager.closePreparedStatement(threadName, getTableName());
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/CustomerTableLoader.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/CustomerTableLoader.java
new file mode 100644
index 0000000..588c5ef
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/CustomerTableLoader.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders;
+
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConfig;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConstants;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCUtil;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.BatchProcessor;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.ConnectionManager;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.RetryHandler;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.TPCCLoaderConstants;
+import com.oltpbenchmark.benchmarks.tpcc.pojo.Customer;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import lombok.extern.slf4j.Slf4j;
+
+/** Loader for the CUSTOMER table. */
+@Slf4j
+public class CustomerTableLoader extends AbstractTableLoader {
+
+ public CustomerTableLoader(
+ BenchmarkModule benchmark,
+ ConnectionManager connectionManager,
+ RetryHandler retryHandler,
+ int batchSize) {
+ super(benchmark, connectionManager, retryHandler, batchSize);
+ }
+
+ @Override
+ protected String getTableName() {
+ return TPCCConstants.TABLENAME_CUSTOMER;
+ }
+
+ @Override
+ public void load(String threadName) throws SQLException {
+ throw new UnsupportedOperationException("Use load(String threadName, int warehouseId) instead");
+ }
+
+ /** Loads customers for a specific warehouse. */
+ public void load(String threadName, int warehouseId) throws SQLException {
+ log.info("Starting to load CUSTOMER for warehouse {}", warehouseId);
+ loadCustomers(
+ threadName, warehouseId, TPCCConfig.configDistPerWhse, TPCCConfig.configCustPerDist);
+ log.info("Finished loading CUSTOMER for warehouse {}", warehouseId);
+ }
+
+ private void loadCustomers(
+ String threadName, int warehouseId, int districtsPerWarehouse, int customersPerDistrict)
+ throws SQLException {
+ BatchProcessor batchProcessor =
+ new BatchProcessor<>(batchSize, this::setCustomerParameters);
+
+ for (int d = 1; d <= districtsPerWarehouse; d++) {
+ for (int c = 1; c <= customersPerDistrict; c++) {
+ Customer customer = generateCustomer(warehouseId, d, c);
+
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.add(customer, stmt);
+ },
+ threadName,
+ "Insert Customer");
+ }
+ }
+
+ // Flush any remaining customers
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.flush(stmt);
+ },
+ threadName,
+ "Flush remaining customers");
+ }
+
+ private Customer generateCustomer(int warehouseId, int districtId, int customerId) {
+ Customer customer = new Customer();
+ Timestamp sysdate = new Timestamp(System.currentTimeMillis());
+
+ customer.c_id = customerId;
+ customer.c_d_id = districtId;
+ customer.c_w_id = warehouseId;
+
+ // discount is random between [0.0000 ... 0.5000]
+ customer.c_discount =
+ (float)
+ (TPCCUtil.randomNumber(1, TPCCLoaderConstants.CUSTOMER_DISCOUNT_MAX, benchmark.rng())
+ / 10000.0);
+
+ // 10% Bad Credit, 90% Good Credit
+ if (TPCCUtil.randomNumber(1, 100, benchmark.rng())
+ <= TPCCLoaderConstants.BAD_CREDIT_THRESHOLD) {
+ customer.c_credit = TPCCLoaderConstants.BAD_CREDIT;
+ } else {
+ customer.c_credit = TPCCLoaderConstants.GOOD_CREDIT;
+ }
+
+ // Last name handling - first 1000 customers have special last names
+ if (customerId <= 1000) {
+ customer.c_last = TPCCUtil.getLastName(customerId - 1);
+ } else {
+ customer.c_last = TPCCUtil.getNonUniformRandomLastNameForLoad(benchmark.rng());
+ }
+
+ customer.c_first = TPCCUtil.randomStr(TPCCUtil.randomNumber(8, 16, benchmark.rng()));
+ customer.c_credit_lim = (float) TPCCLoaderConstants.CUSTOMER_CREDIT_LIMIT;
+
+ customer.c_balance = (float) TPCCLoaderConstants.CUSTOMER_INITIAL_BALANCE;
+ customer.c_ytd_payment = (float) TPCCLoaderConstants.CUSTOMER_INITIAL_YTD_PAYMENT;
+ customer.c_payment_cnt = TPCCLoaderConstants.CUSTOMER_INITIAL_PAYMENT_CNT;
+ customer.c_delivery_cnt = TPCCLoaderConstants.CUSTOMER_INITIAL_DELIVERY_CNT;
+
+ customer.c_street_1 = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
+ customer.c_street_2 = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
+ customer.c_city = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
+ customer.c_state = TPCCUtil.randomStr(3).toUpperCase();
+
+ // TPC-C 4.3.2.7: 4 random digits + "11111"
+ customer.c_zip = TPCCUtil.randomNStr(4) + TPCCLoaderConstants.ZIP_SUFFIX;
+ customer.c_phone = TPCCUtil.randomNStr(16);
+ customer.c_since = sysdate;
+ customer.c_middle = TPCCLoaderConstants.MIDDLE_NAME;
+ customer.c_data = TPCCUtil.randomStr(TPCCUtil.randomNumber(300, 500, benchmark.rng()));
+
+ return customer;
+ }
+
+ private void setCustomerParameters(PreparedStatement stmt, Customer customer) {
+ try {
+ int idx = 1;
+ stmt.setLong(idx++, customer.c_w_id);
+ stmt.setLong(idx++, customer.c_d_id);
+ stmt.setLong(idx++, customer.c_id);
+ stmt.setDouble(idx++, customer.c_discount);
+ stmt.setString(idx++, customer.c_credit);
+ stmt.setString(idx++, customer.c_last);
+ stmt.setString(idx++, customer.c_first);
+ stmt.setDouble(idx++, customer.c_credit_lim);
+ stmt.setDouble(idx++, customer.c_balance);
+ stmt.setDouble(idx++, customer.c_ytd_payment);
+ stmt.setLong(idx++, (long) customer.c_payment_cnt);
+ stmt.setLong(idx++, (long) customer.c_delivery_cnt);
+ stmt.setString(idx++, customer.c_street_1);
+ stmt.setString(idx++, customer.c_street_2);
+ stmt.setString(idx++, customer.c_city);
+ stmt.setString(idx++, customer.c_state);
+ stmt.setString(idx++, customer.c_zip);
+ stmt.setString(idx++, customer.c_phone);
+ stmt.setTimestamp(idx++, customer.c_since);
+ stmt.setString(idx++, customer.c_middle);
+ stmt.setString(idx, customer.c_data);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to set customer parameters", e);
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/DistrictTableLoader.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/DistrictTableLoader.java
new file mode 100644
index 0000000..a63add9
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/DistrictTableLoader.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders;
+
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConfig;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConstants;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCUtil;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.ConnectionManager;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.RetryHandler;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.TPCCLoaderConstants;
+import com.oltpbenchmark.benchmarks.tpcc.pojo.District;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import lombok.extern.slf4j.Slf4j;
+
+/** Loader for the DISTRICT table. */
+@Slf4j
+public class DistrictTableLoader extends AbstractTableLoader {
+
+ public DistrictTableLoader(
+ BenchmarkModule benchmark,
+ ConnectionManager connectionManager,
+ RetryHandler retryHandler,
+ int batchSize) {
+ super(benchmark, connectionManager, retryHandler, batchSize);
+ }
+
+ @Override
+ protected String getTableName() {
+ return TPCCConstants.TABLENAME_DISTRICT;
+ }
+
+ @Override
+ public void load(String threadName) throws SQLException {
+ throw new UnsupportedOperationException("Use load(String threadName, int warehouseId) instead");
+ }
+
+ /** Loads districts for a specific warehouse. */
+ public void load(String threadName, int warehouseId) throws SQLException {
+ log.info("Starting to load DISTRICT for warehouse {}", warehouseId);
+
+ loadDistricts(threadName, warehouseId, TPCCConfig.configDistPerWhse);
+
+ log.info("Finished loading DISTRICT for warehouse {}", warehouseId);
+ }
+
+ private void loadDistricts(String threadName, int warehouseId, int districtsPerWarehouse)
+ throws SQLException {
+ for (int d = 1; d <= districtsPerWarehouse; d++) {
+ District district = generateDistrict(warehouseId, d);
+
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ setDistrictParameters(stmt, district);
+ stmt.executeUpdate();
+ },
+ threadName,
+ "Insert District");
+ }
+ }
+
+ private District generateDistrict(int warehouseId, int districtId) {
+ District district = new District();
+
+ district.d_id = districtId;
+ district.d_w_id = warehouseId;
+ district.d_ytd = (float) TPCCLoaderConstants.DISTRICT_INITIAL_YTD;
+
+ // random within [0.0000 .. 0.2000]
+ district.d_tax =
+ (float)
+ (TPCCUtil.randomNumber(0, TPCCLoaderConstants.DISTRICT_TAX_MAX, benchmark.rng())
+ / 10000.0);
+
+ district.d_next_o_id = TPCCConfig.configCustPerDist + 1;
+ district.d_name = TPCCUtil.randomStr(TPCCUtil.randomNumber(6, 10, benchmark.rng()));
+ district.d_street_1 = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
+ district.d_street_2 = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
+ district.d_city = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
+ district.d_state = TPCCUtil.randomStr(3).toUpperCase();
+ district.d_zip = "123456789";
+
+ return district;
+ }
+
+ private void setDistrictParameters(PreparedStatement stmt, District district)
+ throws SQLException {
+ int idx = 1;
+ stmt.setLong(idx++, district.d_w_id);
+ stmt.setLong(idx++, district.d_id);
+ stmt.setDouble(idx++, district.d_ytd);
+ stmt.setDouble(idx++, district.d_tax);
+ stmt.setLong(idx++, district.d_next_o_id);
+ stmt.setString(idx++, district.d_name);
+ stmt.setString(idx++, district.d_street_1);
+ stmt.setString(idx++, district.d_street_2);
+ stmt.setString(idx++, district.d_city);
+ stmt.setString(idx++, district.d_state);
+ stmt.setString(idx, district.d_zip);
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/HistoryTableLoader.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/HistoryTableLoader.java
new file mode 100644
index 0000000..4f7ecca
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/HistoryTableLoader.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders;
+
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConfig;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConstants;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCUtil;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.BatchProcessor;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.ConnectionManager;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.RetryHandler;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.TPCCLoaderConstants;
+import com.oltpbenchmark.benchmarks.tpcc.pojo.History;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import lombok.extern.slf4j.Slf4j;
+
+/** Loader for the HISTORY table. */
+@Slf4j
+public class HistoryTableLoader extends AbstractTableLoader {
+
+ public HistoryTableLoader(
+ BenchmarkModule benchmark,
+ ConnectionManager connectionManager,
+ RetryHandler retryHandler,
+ int batchSize) {
+ super(benchmark, connectionManager, retryHandler, batchSize);
+ }
+
+ @Override
+ protected String getTableName() {
+ return TPCCConstants.TABLENAME_HISTORY;
+ }
+
+ @Override
+ public void load(String threadName) throws SQLException {
+ throw new UnsupportedOperationException("Use load(String threadName, int warehouseId) instead");
+ }
+
+ /** Loads customer history for a specific warehouse. */
+ public void load(String threadName, int warehouseId) throws SQLException {
+ log.info("Starting to load HISTORY for warehouse {}", warehouseId);
+
+ loadCustomerHistory(
+ threadName, warehouseId, TPCCConfig.configDistPerWhse, TPCCConfig.configCustPerDist);
+
+ log.info("Finished loading HISTORY for warehouse {}", warehouseId);
+ }
+
+ private void loadCustomerHistory(
+ String threadName, int warehouseId, int districtsPerWarehouse, int customersPerDistrict)
+ throws SQLException {
+ BatchProcessor batchProcessor =
+ new BatchProcessor<>(batchSize, this::setHistoryParameters);
+
+ for (int d = 1; d <= districtsPerWarehouse; d++) {
+ for (int c = 1; c <= customersPerDistrict; c++) {
+ History history = generateHistory(warehouseId, d, c);
+
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.add(history, stmt);
+ },
+ threadName,
+ "Insert History");
+ }
+ }
+
+ // Flush any remaining history records
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.flush(stmt);
+ },
+ threadName,
+ "Flush remaining history");
+ }
+
+ private History generateHistory(int warehouseId, int districtId, int customerId) {
+ History history = new History();
+ Timestamp sysdate = new Timestamp(System.currentTimeMillis());
+
+ history.h_c_id = customerId;
+ history.h_c_d_id = districtId;
+ history.h_c_w_id = warehouseId;
+ history.h_d_id = districtId;
+ history.h_w_id = warehouseId;
+ history.h_date = sysdate;
+ history.h_amount = (float) TPCCLoaderConstants.HISTORY_AMOUNT;
+ history.h_data =
+ TPCCUtil.randomStr(
+ TPCCUtil.randomNumber(
+ TPCCLoaderConstants.HISTORY_DATA_MIN_LENGTH,
+ TPCCLoaderConstants.HISTORY_DATA_MAX_LENGTH,
+ benchmark.rng()));
+
+ return history;
+ }
+
+ private void setHistoryParameters(PreparedStatement stmt, History history) {
+ try {
+ int idx = 1;
+ stmt.setInt(idx++, history.h_c_id);
+ stmt.setInt(idx++, history.h_c_d_id);
+ stmt.setInt(idx++, history.h_c_w_id);
+ stmt.setInt(idx++, history.h_d_id);
+ stmt.setInt(idx++, history.h_w_id);
+ stmt.setTimestamp(idx++, history.h_date);
+ stmt.setDouble(idx++, history.h_amount);
+ stmt.setString(idx, history.h_data);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to set history parameters", e);
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/ItemTableLoader.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/ItemTableLoader.java
new file mode 100644
index 0000000..0345705
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/ItemTableLoader.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders;
+
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConfig;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConstants;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCUtil;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.BatchProcessor;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.ConnectionManager;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.RetryHandler;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.TPCCLoaderConstants;
+import com.oltpbenchmark.benchmarks.tpcc.pojo.Item;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import lombok.extern.slf4j.Slf4j;
+
+/** Loader for the ITEM table. */
+@Slf4j
+public class ItemTableLoader extends AbstractTableLoader {
+
+ public ItemTableLoader(
+ BenchmarkModule benchmark,
+ ConnectionManager connectionManager,
+ RetryHandler retryHandler,
+ int batchSize) {
+ super(benchmark, connectionManager, retryHandler, batchSize);
+ }
+
+ @Override
+ protected String getTableName() {
+ return TPCCConstants.TABLENAME_ITEM;
+ }
+
+ @Override
+ public void load(String threadName) throws SQLException {
+ log.info("Starting to load ITEM table");
+
+ loadItems(threadName, TPCCConfig.configItemCount);
+
+ log.info("Finished loading ITEM table");
+ }
+
+ private void loadItems(String threadName, int itemCount) throws SQLException {
+ BatchProcessor
- batchProcessor = new BatchProcessor<>(batchSize, this::setItemParameters);
+
+ for (int i = 1; i <= itemCount; i++) {
+ Item item = generateItem(i);
+
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.add(item, stmt);
+ },
+ threadName,
+ "Insert Item");
+ }
+
+ // Flush any remaining items
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.flush(stmt);
+ },
+ threadName,
+ "Flush remaining items");
+ }
+
+ private Item generateItem(int itemId) {
+ Item item = new Item();
+ item.i_id = itemId;
+ item.i_name =
+ TPCCUtil.randomStr(
+ TPCCUtil.randomNumber(
+ TPCCLoaderConstants.ITEM_NAME_MIN_LENGTH,
+ TPCCLoaderConstants.ITEM_NAME_MAX_LENGTH,
+ benchmark.rng()));
+ item.i_price =
+ TPCCUtil.randomNumber(
+ TPCCLoaderConstants.ITEM_PRICE_MIN,
+ TPCCLoaderConstants.ITEM_PRICE_MAX,
+ benchmark.rng())
+ / 100.0;
+
+ // Generate i_data
+ item.i_data = generateItemData();
+ item.i_im_id = TPCCUtil.randomNumber(1, 10000, benchmark.rng());
+
+ return item;
+ }
+
+ private String generateItemData() {
+ int randPct = TPCCUtil.randomNumber(1, 100, benchmark.rng());
+ int len =
+ TPCCUtil.randomNumber(
+ TPCCLoaderConstants.ITEM_DATA_MIN_LENGTH,
+ TPCCLoaderConstants.ITEM_DATA_MAX_LENGTH,
+ benchmark.rng());
+
+ if (randPct > TPCCLoaderConstants.ORIGINAL_DATA_THRESHOLD) {
+ // 90% of time i_data is a random string
+ return TPCCUtil.randomStr(len);
+ } else {
+ // 10% of time i_data has "ORIGINAL" in the middle
+ int startOriginal = TPCCUtil.randomNumber(2, len - 8, benchmark.rng());
+ return TPCCUtil.randomStr(startOriginal - 1)
+ + TPCCLoaderConstants.ORIGINAL_STRING
+ + TPCCUtil.randomStr(len - startOriginal - 9);
+ }
+ }
+
+ private void setItemParameters(PreparedStatement stmt, Item item) {
+ try {
+ int idx = 1;
+ stmt.setLong(idx++, item.i_id);
+ stmt.setString(idx++, item.i_name);
+ stmt.setDouble(idx++, item.i_price);
+ stmt.setString(idx++, item.i_data);
+ stmt.setLong(idx, item.i_im_id);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to set item parameters", e);
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/NewOrderTableLoader.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/NewOrderTableLoader.java
new file mode 100644
index 0000000..d01c5d0
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/NewOrderTableLoader.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders;
+
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConfig;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConstants;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.BatchProcessor;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.ConnectionManager;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.RetryHandler;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.TPCCLoaderConstants;
+import com.oltpbenchmark.benchmarks.tpcc.pojo.NewOrder;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Loader for the NEW_ORDER table.
+ *
+ *
New orders are only created for unprocessed orders (o_id >= FIRST_UNPROCESSED_O_ID).
+ */
+@Slf4j
+public class NewOrderTableLoader extends AbstractTableLoader {
+
+ public NewOrderTableLoader(
+ BenchmarkModule benchmark,
+ ConnectionManager connectionManager,
+ RetryHandler retryHandler,
+ int batchSize) {
+ super(benchmark, connectionManager, retryHandler, batchSize);
+ }
+
+ @Override
+ protected String getTableName() {
+ return TPCCConstants.TABLENAME_NEWORDER;
+ }
+
+ @Override
+ public void load(String threadName) throws SQLException {
+ throw new UnsupportedOperationException("Use load(String threadName, int warehouseId) instead");
+ }
+
+ /**
+ * Loads new orders for a specific warehouse.
+ *
+ *
According to TPC-C specification, new orders are only created for orders with o_id >=
+ * FIRST_UNPROCESSED_O_ID (2101).
+ */
+ public void load(String threadName, int warehouseId) throws SQLException {
+ log.info("Starting to load NEW_ORDER for warehouse {}", warehouseId);
+
+ loadNewOrders(
+ threadName, warehouseId, TPCCConfig.configDistPerWhse, TPCCConfig.configCustPerDist);
+
+ log.info("Finished loading NEW_ORDER for warehouse {}", warehouseId);
+ }
+
+ private void loadNewOrders(
+ String threadName, int warehouseId, int districtsPerWarehouse, int customersPerDistrict)
+ throws SQLException {
+ BatchProcessor batchProcessor =
+ new BatchProcessor<>(batchSize, this::setNewOrderParameters);
+
+ for (int d = 1; d <= districtsPerWarehouse; d++) {
+ for (int c = 1; c <= customersPerDistrict; c++) {
+ // New orders are only created for unprocessed orders
+ if (c >= TPCCLoaderConstants.FIRST_UNPROCESSED_O_ID) {
+ NewOrder newOrder = new NewOrder();
+ newOrder.no_w_id = warehouseId;
+ newOrder.no_d_id = d;
+ newOrder.no_o_id = c;
+
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.add(newOrder, stmt);
+ },
+ threadName,
+ "Insert New Order");
+ }
+ }
+ }
+
+ // Flush any remaining new orders
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.flush(stmt);
+ },
+ threadName,
+ "Flush remaining new orders");
+ }
+
+ private void setNewOrderParameters(PreparedStatement stmt, NewOrder newOrder) {
+ try {
+ int idx = 1;
+ stmt.setInt(idx++, newOrder.no_w_id);
+ stmt.setInt(idx++, newOrder.no_d_id);
+ stmt.setInt(idx, newOrder.no_o_id);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to set new order parameters", e);
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/OrderLineTableLoader.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/OrderLineTableLoader.java
new file mode 100644
index 0000000..6dd747c
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/OrderLineTableLoader.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders;
+
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConfig;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConstants;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCUtil;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.BatchProcessor;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.ConnectionManager;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.RetryHandler;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.TPCCLoaderConstants;
+import com.oltpbenchmark.benchmarks.tpcc.pojo.Customer;
+import com.oltpbenchmark.benchmarks.tpcc.pojo.OrderLine;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.sql.Types;
+import java.util.Random;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Loader for the ORDER_LINE table.
+ *
+ * Order lines are created for each order, with the number of lines determined by the order's
+ * line count.
+ */
+@Slf4j
+public class OrderLineTableLoader extends AbstractTableLoader {
+
+ public OrderLineTableLoader(
+ BenchmarkModule benchmark,
+ ConnectionManager connectionManager,
+ RetryHandler retryHandler,
+ int batchSize) {
+ super(benchmark, connectionManager, retryHandler, batchSize);
+ }
+
+ @Override
+ protected String getTableName() {
+ return TPCCConstants.TABLENAME_ORDERLINE;
+ }
+
+ @Override
+ public void load(String threadName) throws SQLException {
+ throw new UnsupportedOperationException("Use load(String threadName, int warehouseId) instead");
+ }
+
+ /** Loads order lines for a specific warehouse. */
+ public void load(String threadName, int warehouseId) throws SQLException {
+ log.info("Starting to load ORDER_LINE for warehouse {}", warehouseId);
+
+ loadOrderLines(
+ threadName, warehouseId, TPCCConfig.configDistPerWhse, TPCCConfig.configCustPerDist);
+
+ log.info("Finished loading ORDER_LINE for warehouse {}", warehouseId);
+ }
+
+ private void loadOrderLines(
+ String threadName, int warehouseId, int districtsPerWarehouse, int customersPerDistrict)
+ throws SQLException {
+ BatchProcessor batchProcessor =
+ new BatchProcessor<>(batchSize, this::setOrderLineParameters);
+
+ for (int d = 1; d <= districtsPerWarehouse; d++) {
+ for (int c = 1; c <= customersPerDistrict; c++) {
+ int orderLineCount = getOrderLineCount(warehouseId, c, d);
+
+ for (int l = 1; l <= orderLineCount; l++) {
+ OrderLine orderLine = generateOrderLine(warehouseId, d, c, l);
+
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.add(orderLine, stmt);
+ },
+ threadName,
+ "Insert Order Line");
+ }
+ }
+ }
+
+ // Flush any remaining order lines
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.flush(stmt);
+ },
+ threadName,
+ "Flush remaining order lines");
+ }
+
+ private OrderLine generateOrderLine(
+ int warehouseId, int districtId, int orderId, int lineNumber) {
+ OrderLine orderLine = new OrderLine();
+
+ orderLine.ol_w_id = warehouseId;
+ orderLine.ol_d_id = districtId;
+ orderLine.ol_o_id = orderId;
+ orderLine.ol_number = lineNumber;
+ orderLine.ol_i_id = TPCCUtil.randomNumber(1, TPCCConfig.configItemCount, benchmark.rng());
+
+ // Set delivery date and amount based on whether order is processed
+ if (orderId < TPCCLoaderConstants.FIRST_UNPROCESSED_O_ID) {
+ // Processed order
+ orderLine.ol_delivery_d = new Timestamp(System.currentTimeMillis());
+ orderLine.ol_amount = 0;
+ } else {
+ // Unprocessed order
+ orderLine.ol_delivery_d = null;
+ // Random amount within [0.01 .. 9,999.99]
+ orderLine.ol_amount = (float) (TPCCUtil.randomNumber(1, 999999, benchmark.rng()) / 100.0);
+ }
+
+ orderLine.ol_supply_w_id = orderLine.ol_w_id;
+ orderLine.ol_quantity = TPCCLoaderConstants.ORDER_LINE_QUANTITY;
+ orderLine.ol_dist_info = TPCCUtil.randomStr(24);
+
+ return orderLine;
+ }
+
+ private int getOrderLineCount(int warehouseId, int orderId, int districtId) {
+ // Use a deterministic random based on customer info for consistency
+ // This ensures the same order always has the same number of order lines
+ Customer customer = new Customer();
+ customer.c_id = orderId;
+ customer.c_d_id = districtId;
+ customer.c_w_id = warehouseId;
+
+ Random random = new Random(customer.hashCode());
+ return TPCCUtil.randomNumber(
+ TPCCLoaderConstants.ORDER_LINE_COUNT_MIN, TPCCLoaderConstants.ORDER_LINE_COUNT_MAX, random);
+ }
+
+ private void setOrderLineParameters(PreparedStatement stmt, OrderLine orderLine) {
+ try {
+ int idx = 1;
+ stmt.setInt(idx++, orderLine.ol_w_id);
+ stmt.setInt(idx++, orderLine.ol_d_id);
+ stmt.setInt(idx++, orderLine.ol_o_id);
+ stmt.setInt(idx++, orderLine.ol_number);
+ stmt.setInt(idx++, orderLine.ol_i_id);
+
+ if (orderLine.ol_delivery_d != null) {
+ stmt.setTimestamp(idx++, orderLine.ol_delivery_d);
+ } else {
+ stmt.setNull(idx++, Types.TIMESTAMP);
+ }
+
+ stmt.setFloat(idx++, orderLine.ol_amount);
+ stmt.setInt(idx++, orderLine.ol_supply_w_id);
+ stmt.setInt(idx++, orderLine.ol_quantity);
+ stmt.setString(idx, orderLine.ol_dist_info);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to set order line parameters", e);
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/OrderTableLoader.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/OrderTableLoader.java
new file mode 100644
index 0000000..6ff5f92
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/OrderTableLoader.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders;
+
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConfig;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConstants;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCUtil;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.BatchProcessor;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.ConnectionManager;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.RetryHandler;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.TPCCLoaderConstants;
+import com.oltpbenchmark.benchmarks.tpcc.pojo.Customer;
+import com.oltpbenchmark.benchmarks.tpcc.pojo.Oorder;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.sql.Types;
+import java.util.Random;
+import lombok.extern.slf4j.Slf4j;
+
+/** Loader for the OORDER (open order) table. */
+@Slf4j
+public class OrderTableLoader extends AbstractTableLoader {
+
+ public OrderTableLoader(
+ BenchmarkModule benchmark,
+ ConnectionManager connectionManager,
+ RetryHandler retryHandler,
+ int batchSize) {
+ super(benchmark, connectionManager, retryHandler, batchSize);
+ }
+
+ @Override
+ protected String getTableName() {
+ return TPCCConstants.TABLENAME_OPENORDER;
+ }
+
+ @Override
+ public void load(String threadName) throws SQLException {
+ throw new UnsupportedOperationException("Use load(String threadName, int warehouseId) instead");
+ }
+
+ /** Loads open orders for a specific warehouse. */
+ public void load(String threadName, int warehouseId) throws SQLException {
+ log.info("Starting to load OORDER for warehouse {}", warehouseId);
+
+ loadOpenOrders(
+ threadName, warehouseId, TPCCConfig.configDistPerWhse, TPCCConfig.configCustPerDist);
+
+ log.info("Finished loading OORDER for warehouse {}", warehouseId);
+ }
+
+ private void loadOpenOrders(
+ String threadName, int warehouseId, int districtsPerWarehouse, int customersPerDistrict)
+ throws SQLException {
+ BatchProcessor batchProcessor =
+ new BatchProcessor<>(batchSize, this::setOrderParameters);
+
+ for (int d = 1; d <= districtsPerWarehouse; d++) {
+ // TPC-C 4.3.3.1: o_c_id must be a permutation of [1, 3000]
+ int[] c_ids = generateCustomerIdPermutation(customersPerDistrict);
+
+ for (int c = 1; c <= customersPerDistrict; c++) {
+ Oorder order = generateOrder(warehouseId, d, c, c_ids[c - 1]);
+
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.add(order, stmt);
+ },
+ threadName,
+ "Insert Order");
+ }
+ }
+
+ // Flush any remaining orders
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.flush(stmt);
+ },
+ threadName,
+ "Flush remaining orders");
+ }
+
+ private int[] generateCustomerIdPermutation(int customersPerDistrict) {
+ int[] c_ids = new int[customersPerDistrict];
+ for (int i = 0; i < customersPerDistrict; i++) {
+ c_ids[i] = i + 1;
+ }
+
+ // Fisher-Yates shuffle
+ for (int i = 0; i < c_ids.length - 1; i++) {
+ int remaining = c_ids.length - i - 1;
+ int swapIndex = benchmark.rng().nextInt(remaining) + i + 1;
+
+ int temp = c_ids[swapIndex];
+ c_ids[swapIndex] = c_ids[i];
+ c_ids[i] = temp;
+ }
+
+ return c_ids;
+ }
+
+ private Oorder generateOrder(int warehouseId, int districtId, int orderId, int customerId) {
+ Oorder order = new Oorder();
+
+ order.o_id = orderId;
+ order.o_w_id = warehouseId;
+ order.o_d_id = districtId;
+ order.o_c_id = customerId;
+
+ // o_carrier_id is set *only* for orders with ids < 2101 [4.3.3.1]
+ if (order.o_id < TPCCLoaderConstants.FIRST_UNPROCESSED_O_ID) {
+ order.o_carrier_id =
+ TPCCUtil.randomNumber(
+ TPCCLoaderConstants.CARRIER_ID_MIN,
+ TPCCLoaderConstants.CARRIER_ID_MAX,
+ benchmark.rng());
+ } else {
+ order.o_carrier_id = null;
+ }
+
+ order.o_ol_cnt = getRandomOrderLineCount(warehouseId, orderId, districtId);
+ order.o_all_local = 1;
+ order.o_entry_d = new Timestamp(System.currentTimeMillis());
+
+ return order;
+ }
+
+ private int getRandomOrderLineCount(int warehouseId, int orderId, int districtId) {
+ // Use a deterministic random based on customer info for consistency
+ Customer customer = new Customer();
+ customer.c_id = orderId;
+ customer.c_d_id = districtId;
+ customer.c_w_id = warehouseId;
+
+ Random random = new Random(customer.hashCode());
+ return TPCCUtil.randomNumber(
+ TPCCLoaderConstants.ORDER_LINE_COUNT_MIN, TPCCLoaderConstants.ORDER_LINE_COUNT_MAX, random);
+ }
+
+ private void setOrderParameters(PreparedStatement stmt, Oorder order) {
+ try {
+ int idx = 1;
+ stmt.setInt(idx++, order.o_w_id);
+ stmt.setInt(idx++, order.o_d_id);
+ stmt.setInt(idx++, order.o_id);
+ stmt.setInt(idx++, order.o_c_id);
+
+ if (order.o_carrier_id != null) {
+ stmt.setInt(idx++, order.o_carrier_id);
+ } else {
+ stmt.setNull(idx++, Types.INTEGER);
+ }
+
+ stmt.setInt(idx++, order.o_ol_cnt);
+ stmt.setInt(idx++, order.o_all_local);
+ stmt.setTimestamp(idx, order.o_entry_d);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to set order parameters", e);
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/StockTableLoader.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/StockTableLoader.java
new file mode 100644
index 0000000..78af02e
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/StockTableLoader.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders;
+
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConfig;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConstants;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCUtil;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.BatchProcessor;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.ConnectionManager;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.RetryHandler;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.TPCCLoaderConstants;
+import com.oltpbenchmark.benchmarks.tpcc.pojo.Stock;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import lombok.extern.slf4j.Slf4j;
+
+/** Loader for the STOCK table. */
+@Slf4j
+public class StockTableLoader extends AbstractTableLoader {
+
+ public StockTableLoader(
+ BenchmarkModule benchmark,
+ ConnectionManager connectionManager,
+ RetryHandler retryHandler,
+ int batchSize) {
+ super(benchmark, connectionManager, retryHandler, batchSize);
+ }
+
+ @Override
+ protected String getTableName() {
+ return TPCCConstants.TABLENAME_STOCK;
+ }
+
+ @Override
+ public void load(String threadName) throws SQLException {
+ throw new UnsupportedOperationException("Use load(String threadName, int warehouseId) instead");
+ }
+
+ /** Loads stock for a specific warehouse. */
+ public void load(String threadName, int warehouseId) throws SQLException {
+ log.info("Starting to load STOCK for warehouse {}", warehouseId);
+
+ loadStock(threadName, warehouseId, TPCCConfig.configItemCount);
+
+ log.info("Finished loading STOCK for warehouse {}", warehouseId);
+ }
+
+ private void loadStock(String threadName, int warehouseId, int numItems) throws SQLException {
+ BatchProcessor batchProcessor =
+ new BatchProcessor<>(batchSize, this::setStockParameters);
+
+ for (int i = 1; i <= numItems; i++) {
+ Stock stock = generateStock(warehouseId, i);
+
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.add(stock, stmt);
+ },
+ threadName,
+ "Insert Stock");
+ }
+
+ // Flush any remaining stocks
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ batchProcessor.flush(stmt);
+ },
+ threadName,
+ "Flush remaining stocks");
+ }
+
+ private Stock generateStock(int warehouseId, int itemId) {
+ Stock stock = new Stock();
+
+ stock.s_i_id = itemId;
+ stock.s_w_id = warehouseId;
+ stock.s_quantity =
+ TPCCUtil.randomNumber(
+ TPCCLoaderConstants.STOCK_QUANTITY_MIN,
+ TPCCLoaderConstants.STOCK_QUANTITY_MAX,
+ benchmark.rng());
+ stock.s_ytd = 0;
+ stock.s_order_cnt = 0;
+ stock.s_remote_cnt = 0;
+
+ // Generate s_data (similar to item data)
+ stock.s_data = generateStockData();
+
+ return stock;
+ }
+
+ private String generateStockData() {
+ int randPct = TPCCUtil.randomNumber(1, 100, benchmark.rng());
+ int len =
+ TPCCUtil.randomNumber(
+ TPCCLoaderConstants.ITEM_DATA_MIN_LENGTH,
+ TPCCLoaderConstants.ITEM_DATA_MAX_LENGTH,
+ benchmark.rng());
+
+ if (randPct > TPCCLoaderConstants.ORIGINAL_DATA_THRESHOLD) {
+ // 90% of time s_data is a random string
+ return TPCCUtil.randomStr(len);
+ } else {
+ // 10% of time s_data has "ORIGINAL" in the middle
+ int startOriginal = TPCCUtil.randomNumber(2, len - 8, benchmark.rng());
+ return TPCCUtil.randomStr(startOriginal - 1)
+ + TPCCLoaderConstants.ORIGINAL_STRING
+ + TPCCUtil.randomStr(len - startOriginal - 9);
+ }
+ }
+
+ private void setStockParameters(PreparedStatement stmt, Stock stock) {
+ try {
+ int idx = 1;
+ stmt.setLong(idx++, stock.s_w_id);
+ stmt.setLong(idx++, stock.s_i_id);
+ stmt.setLong(idx++, stock.s_quantity);
+ stmt.setDouble(idx++, stock.s_ytd);
+ stmt.setLong(idx++, stock.s_order_cnt);
+ stmt.setLong(idx++, stock.s_remote_cnt);
+ stmt.setString(idx++, stock.s_data);
+
+ // Set 10 district fields (s_dist_01 through s_dist_10)
+ for (int i = 0; i < 10; i++) {
+ stmt.setString(idx++, TPCCUtil.randomStr(24));
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to set stock parameters", e);
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/WarehouseTableLoader.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/WarehouseTableLoader.java
new file mode 100644
index 0000000..7c56662
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/custom/auroradsql/loaders/WarehouseTableLoader.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.loaders;
+
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCConstants;
+import com.oltpbenchmark.benchmarks.tpcc.TPCCUtil;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.ConnectionManager;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.RetryHandler;
+import com.oltpbenchmark.benchmarks.tpcc.custom.auroradsql.TPCCLoaderConstants;
+import com.oltpbenchmark.benchmarks.tpcc.pojo.Warehouse;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import lombok.extern.slf4j.Slf4j;
+
+/** Loader for the WAREHOUSE table. */
+@Slf4j
+public class WarehouseTableLoader extends AbstractTableLoader {
+
+ public WarehouseTableLoader(
+ BenchmarkModule benchmark,
+ ConnectionManager connectionManager,
+ RetryHandler retryHandler,
+ int batchSize) {
+ super(benchmark, connectionManager, retryHandler, batchSize);
+ }
+
+ @Override
+ protected String getTableName() {
+ return TPCCConstants.TABLENAME_WAREHOUSE;
+ }
+
+ @Override
+ public void load(String threadName) throws SQLException {
+ throw new UnsupportedOperationException("Use load(String threadName, int warehouseId) instead");
+ }
+
+ /** Loads a single warehouse. */
+ public void load(String threadName, int warehouseId) throws SQLException {
+ log.info("Starting to load WAREHOUSE {}", warehouseId);
+
+ loadWarehouse(threadName, warehouseId);
+
+ log.info("Finished loading WAREHOUSE {}", warehouseId);
+ }
+
+ private void loadWarehouse(String threadName, int warehouseId) throws SQLException {
+ Warehouse warehouse = generateWarehouse(warehouseId);
+
+ executeWithRetry(
+ () -> {
+ PreparedStatement stmt = getInsertStatement(threadName);
+ setWarehouseParameters(stmt, warehouse);
+ stmt.execute();
+ },
+ threadName,
+ "Insert Warehouse");
+ }
+
+ private Warehouse generateWarehouse(int warehouseId) {
+ Warehouse warehouse = new Warehouse();
+
+ warehouse.w_id = warehouseId;
+ warehouse.w_ytd = (float) TPCCLoaderConstants.WAREHOUSE_INITIAL_YTD;
+
+ // random within [0.0000 .. 0.2000]
+ warehouse.w_tax =
+ TPCCUtil.randomNumber(0, TPCCLoaderConstants.WAREHOUSE_TAX_MAX, benchmark.rng()) / 10000.0;
+ warehouse.w_name = TPCCUtil.randomStr(TPCCUtil.randomNumber(6, 10, benchmark.rng()));
+ warehouse.w_street_1 = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
+ warehouse.w_street_2 = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
+ warehouse.w_city = TPCCUtil.randomStr(TPCCUtil.randomNumber(10, 20, benchmark.rng()));
+ warehouse.w_state = TPCCUtil.randomStr(3).toUpperCase();
+ warehouse.w_zip = "123456789";
+
+ return warehouse;
+ }
+
+ private void setWarehouseParameters(PreparedStatement stmt, Warehouse warehouse)
+ throws SQLException {
+ int idx = 1;
+ stmt.setLong(idx++, warehouse.w_id);
+ stmt.setDouble(idx++, warehouse.w_ytd);
+ stmt.setDouble(idx++, warehouse.w_tax);
+ stmt.setString(idx++, warehouse.w_name);
+ stmt.setString(idx++, warehouse.w_street_1);
+ stmt.setString(idx++, warehouse.w_street_2);
+ stmt.setString(idx++, warehouse.w_city);
+ stmt.setString(idx++, warehouse.w_state);
+ stmt.setString(idx, warehouse.w_zip);
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/Delivery.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/Delivery.java
index fddaf5a..9cc9b26 100644
--- a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/Delivery.java
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/Delivery.java
@@ -36,6 +36,36 @@ public class Delivery extends TPCCProcedure {
private static final Logger LOG = LoggerFactory.getLogger(Delivery.class);
+ private static final String TX_NAME = "Delivery";
+ private static final String GET_ORDER_ID_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getOrderId" + TPCCConstants.SEPARATOR;
+ private static final String GET_ORDER_ID_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getOrderIdZeroResult";
+ private static final String NEW_ORDER_DELETE_NOT_ONE_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "newOrderDeleteNotOneResult";
+ private static final String UPDATE_CARRIER_ID_NOT_ONE_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateCarrierIdNotOneResult";
+ private static final String UPDATE_BALANCE_DELIVERY_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateBalanceDeliveryZeroResult";
+ private static final String UPDATE_DELIVERY_DATE_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateDeliveryDateZeroResult";
+ private static final String DELETE_ORDER_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "deleteOrder" + TPCCConstants.SEPARATOR;
+ private static final String GET_CUSTOMER_ID_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCustomerId" + TPCCConstants.SEPARATOR;
+ private static final String GET_CUSTOMER_ID_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCustomerIdZeroResult";
+ private static final String UPDATE_CARRIER_ID_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateCarrierId" + TPCCConstants.SEPARATOR;
+ private static final String UPDATE_DELIVERY_DATE_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateDeliveryDate" + TPCCConstants.SEPARATOR;
+ private static final String GET_ORDER_LINE_TOTAL_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getOrderLineTotal" + TPCCConstants.SEPARATOR;
+ private static final String GET_ORDER_LINE_TOTAL_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getOrderLineTotalZeroResult";
+ private static final String UPDATE_BALANCE_DELIVERY_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateBalanceAndDelivery" + TPCCConstants.SEPARATOR;
+
public SQLStmt delivGetOrderIdSQL =
new SQLStmt(
"""
@@ -129,25 +159,29 @@ public void run(
int[] orderIDs = new int[10];
for (d_id = 1; d_id <= terminalDistrictUpperID; d_id++) {
- Integer no_o_id = getOrderId(conn, w_id, d_id);
+ // To fix local variables referenced from a lambda expression must be final or effectively
+ // final
+ final int d_id_local = d_id;
+
+ Integer no_o_id = getOrderId(conn, w_id, d_id_local);
if (no_o_id == null) {
continue;
}
- orderIDs[d_id - 1] = no_o_id;
+ orderIDs[d_id_local - 1] = no_o_id;
- deleteOrder(conn, w_id, d_id, no_o_id);
+ deleteOrder(conn, w_id, d_id_local, no_o_id);
- int customerId = getCustomerId(conn, w_id, d_id, no_o_id);
+ int customerId = getCustomerId(conn, w_id, d_id_local, no_o_id);
- updateCarrierId(conn, w_id, o_carrier_id, d_id, no_o_id);
+ updateCarrierId(conn, w_id, o_carrier_id, d_id_local, no_o_id);
- updateDeliveryDate(conn, w_id, d_id, no_o_id);
+ updateDeliveryDate(conn, w_id, d_id_local, no_o_id);
- float orderLineTotal = getOrderLineTotal(conn, w_id, d_id, no_o_id);
+ float orderLineTotal = getOrderLineTotal(conn, w_id, d_id_local, no_o_id);
- updateBalanceAndDelivery(conn, w_id, d_id, customerId, orderLineTotal);
+ updateBalanceAndDelivery(conn, w_id, d_id_local, customerId, orderLineTotal);
}
if (LOG.isTraceEnabled()) {
@@ -187,7 +221,6 @@ private Integer getOrderId(Connection conn, int w_id, int d_id) throws SQLExcept
if (!rs.next()) {
// This district has no new orders. This can happen but should be rare
-
LOG.warn(String.format("District has no new orders [W_ID=%d, D_ID=%d]", w_id, d_id));
return null;
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/NewOrder.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/NewOrder.java
index ffd44d1..6a0d785 100644
--- a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/NewOrder.java
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/NewOrder.java
@@ -23,6 +23,7 @@
import com.oltpbenchmark.benchmarks.tpcc.TPCCUtil;
import com.oltpbenchmark.benchmarks.tpcc.TPCCWorker;
import com.oltpbenchmark.benchmarks.tpcc.pojo.Stock;
+import com.oltpbenchmark.util.TimeUtil;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@@ -36,6 +37,53 @@ public class NewOrder extends TPCCProcedure {
private static final Logger LOG = LoggerFactory.getLogger(NewOrder.class);
+ private static final String TX_NAME = "NewOrder";
+
+ private static final String GET_CUSTOMER_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCustomer" + TPCCConstants.SEPARATOR;
+ private static final String GET_CUSTOMER_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCustomerZeroResult";
+ private static final String GET_WAREHOUSE_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getWarehouse" + TPCCConstants.SEPARATOR;
+ private static final String GET_WAREHOUSE_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getWarehouseZeroResult";
+ private static final String GET_DISTRICT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getDistrict" + TPCCConstants.SEPARATOR;
+ private static final String GET_DISTRICT_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getDistrictZeroResult";
+ private static final String UPDATE_DISTRICT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateDistrict" + TPCCConstants.SEPARATOR;
+ private static final String UPDATE_DISTRICT_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateDistrictZeroResult";
+ private static final String INSERT_OPEN_ORDER_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "insertOpenOrder" + TPCCConstants.SEPARATOR;
+ private static final String INSERT_OPEN_ORDER_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "insertOpenOrderZeroResult";
+ private static final String INSERT_NEW_ORDER_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "insertNewOrder" + TPCCConstants.SEPARATOR;
+ private static final String INSERT_NEW_ORDER_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "insertNewOrderZeroResult";
+ private static final String UPDATE_STOCK_INSERT_ORDERLINE_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateStockInsertOrderline" + TPCCConstants.SEPARATOR;
+ private static final String UPDATE_STOCK_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateStock" + TPCCConstants.SEPARATOR;
+ private static final String INSERT_ORDERLINE_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "insertOrderline" + TPCCConstants.SEPARATOR;
+ private static final String GET_ITEM_PRICE_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getItemPrice" + TPCCConstants.SEPARATOR;
+ // Tracks the total time taken by all GetItem queries in the transaction
+ private static final String GET_ITEM_PRICE_TOTAL_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getItemPriceTotal" + TPCCConstants.SEPARATOR;
+ private static final String GET_ITEM_PRICE_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getItemPriceZeroResult";
+ private static final String GET_STOCK_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getStock" + TPCCConstants.SEPARATOR;
+ // Tracks the total time taken by all GetStock queries in the transaction
+ private static final String GET_STOCK_TOTAL_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getStockTotal" + TPCCConstants.SEPARATOR;
+ private static final String GET_STOCK_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getStockZeroResult";
+
public final SQLStmt stmtGetCustSQL =
new SQLStmt(
"""
@@ -212,18 +260,30 @@ private void newOrderTransaction(
PreparedStatement stmtInsertOrderLine =
this.getPreparedStatement(conn, stmtInsertOrderLineSQL)) {
+ // Track total time across all gets in the loop.
+ long getItemTotalTime = 0;
+ long getStockTotalTime = 0;
+
for (int ol_number = 1; ol_number <= o_ol_cnt; ol_number++) {
int ol_supply_w_id = supplierWarehouseIDs[ol_number - 1];
int ol_i_id = itemIDs[ol_number - 1];
int ol_quantity = orderQuantities[ol_number - 1];
+ long start = System.nanoTime();
// this may occasionally error and that's ok!
float i_price = getItemPrice(conn, ol_i_id);
+ long end = System.nanoTime();
+ getItemTotalTime += TimeUtil.getTimeDiffInMicro(start, end);
+
float ol_amount = ol_quantity * i_price;
+ start = System.nanoTime();
Stock s = getStock(conn, ol_supply_w_id, ol_i_id, ol_quantity);
+ end = System.nanoTime();
+ getStockTotalTime += TimeUtil.getTimeDiffInMicro(start, end);
+
String ol_dist_info = getDistInfo(d_id, s);
stmtInsertOrderLine.setInt(1, d_next_o_id);
@@ -251,14 +311,16 @@ private void newOrderTransaction(
stmtUpdateStock.setInt(4, ol_i_id);
stmtUpdateStock.setInt(5, ol_supply_w_id);
stmtUpdateStock.addBatch();
- }
+ } // for loop
stmtInsertOrderLine.executeBatch();
+
stmtInsertOrderLine.clearBatch();
stmtUpdateStock.executeBatch();
+
stmtUpdateStock.clearBatch();
- }
+ } // try block
}
private String getDistInfo(int d_id, Stock s) {
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/OrderStatus.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/OrderStatus.java
index 95a08d8..2d7c31c 100644
--- a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/OrderStatus.java
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/OrderStatus.java
@@ -37,6 +37,24 @@ public class OrderStatus extends TPCCProcedure {
private static final Logger LOG = LoggerFactory.getLogger(OrderStatus.class);
+ private static final String TX_NAME = "OrderStatus";
+ private static final String GET_CUSTOMER_BY_NAME_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCustomerByName" + TPCCConstants.SEPARATOR;
+ private static final String GET_CUSTOMER_BY_NAME_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCustomerByNameZeroResult";
+ private static final String GET_CUSTOMER_BY_ID_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCustomerById" + TPCCConstants.SEPARATOR;
+ private static final String GET_CUSTOMER_BY_ID_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCustomerByIdZeroResult";
+ private static final String GET_ORDER_DETAILS_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getOrderDetails" + TPCCConstants.SEPARATOR;
+ private static final String GET_ORDER_DETAILS_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getOrderDetailsZeroResult";
+ private static final String GET_ORDERLINE_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getOrderLines" + TPCCConstants.SEPARATOR;
+ private static final String GET_ORDERLINE_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getOrderLinesZeroResult";
+
public SQLStmt ordStatGetNewestOrdSQL =
new SQLStmt(
"""
@@ -101,14 +119,16 @@ public void run(
int y = TPCCUtil.randomNumber(1, 100, gen);
boolean c_by_name;
- String c_last = null;
- int c_id = -1;
+ final String c_last;
+ final int c_id;
if (y <= 60) {
c_by_name = true;
c_last = TPCCUtil.getNonUniformRandomLastNameForRun(gen);
+ c_id = -1;
} else {
c_by_name = false;
+ c_last = null;
c_id = TPCCUtil.getCustomerID(gen);
}
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/Payment.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/Payment.java
index e2eca95..3aa78e4 100644
--- a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/Payment.java
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/Payment.java
@@ -40,6 +40,46 @@ public class Payment extends TPCCProcedure {
private static final Logger LOG = LoggerFactory.getLogger(Payment.class);
+ private static final String TX_NAME = "Payment";
+ private static final String GET_CUSTOMER_BY_NAME_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCustomerByName" + TPCCConstants.SEPARATOR;
+ private static final String GET_CUSTOMER_BY_NAME_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCustomerByNameZeroResult";
+ private static final String GET_CUSTOMER_BY_ID_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCustomerById" + TPCCConstants.SEPARATOR;
+ private static final String GET_CUSTOMER_BY_ID_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCustomerByIdZeroResult";
+ private static final String GET_WAREHOUSE_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getWarehouse" + TPCCConstants.SEPARATOR;
+ private static final String GET_WAREHOUSE_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getWarehouseZeroResult";
+ private static final String UPDATE_WAREHOUSE_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateWarehouse" + TPCCConstants.SEPARATOR;
+ private static final String UPDATE_WAREHOUSE_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateWarehouseZeroResult";
+ private static final String GET_DISTRICT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getDistrict" + TPCCConstants.SEPARATOR;
+ private static final String GET_DISTRICT_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getDistrictZeroResult";
+ private static final String UPDATE_DISTRICT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateDistrict" + TPCCConstants.SEPARATOR;
+ private static final String UPDATE_DISTRICT_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateDistrictZeroResult";
+ private static final String GET_C_DATA_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCData" + TPCCConstants.SEPARATOR;
+ private static final String GET_C_DATA_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getCDataZeroResult";
+ private static final String UPDATE_BALANCE_C_DATA_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateBalanceCData" + TPCCConstants.SEPARATOR;
+ private static final String UPDATE_BALANCE_C_DATA_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateBalanceCDataZeroResult";
+ private static final String UPDATE_BALANCE_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateBalance" + TPCCConstants.SEPARATOR;
+ private static final String UPDATE_BALANCE_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "updateBalanceZeroResult";
+ private static final String INSERT_HISTORY_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "insertHistory" + TPCCConstants.SEPARATOR;
+
public SQLStmt payUpdateWhseSQL =
new SQLStmt(
"""
@@ -183,6 +223,7 @@ public void run(
if (c.c_credit.equals("BC")) {
// bad credit
+
c.c_data =
getCData(
conn, w_id, districtID, customerDistrictID, customerWarehouseID, paymentAmount, c);
@@ -365,6 +406,7 @@ private Customer getCustomer(
customerDistrictID,
TPCCUtil.getNonUniformRandomLastNameForRun(gen),
conn);
+
} else {
// 40% lookups by customer ID
c =
@@ -611,8 +653,7 @@ public Customer getCustomerByName(
}
// TPC-C 2.5.2.2: Position n / 2 rounded up to the next integer, but
- // that
- // counts starting from 1.
+ // that counts starting from 1.
int index = customers.size() / 2;
if (customers.size() % 2 == 0) {
index -= 1;
diff --git a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/StockLevel.java b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/StockLevel.java
index 23d7767..808a990 100644
--- a/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/StockLevel.java
+++ b/src/main/java/com/oltpbenchmark/benchmarks/tpcc/procedures/StockLevel.java
@@ -33,6 +33,16 @@ public class StockLevel extends TPCCProcedure {
private static final Logger LOG = LoggerFactory.getLogger(StockLevel.class);
+ private static final String TX_NAME = "StockLevel";
+ private static final String GET_ORDER_ID_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getOrderId" + TPCCConstants.SEPARATOR;
+ private static final String GET_ORDER_ID_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getOrderIdZeroResult";
+ private static final String GET_STOCK_COUNT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getStockCount" + TPCCConstants.SEPARATOR;
+ private static final String GET_STOCK_COUNT_ZERO_RESULT_METRIC_NAME =
+ TX_NAME + TPCCConstants.SEPARATOR + "getStockCountZeroResult";
+
public SQLStmt stockGetDistOrderIdSQL =
new SQLStmt(
"""
diff --git a/src/main/java/com/oltpbenchmark/execution/AnonymizationHandler.java b/src/main/java/com/oltpbenchmark/execution/AnonymizationHandler.java
new file mode 100644
index 0000000..86cc9d7
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/execution/AnonymizationHandler.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.execution;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.configuration2.XMLConfiguration;
+
+/** Handles anonymization of datasets using differential privacy. */
+@Slf4j
+public class AnonymizationHandler {
+ /**
+ * Apply anonymization to specified tables with differential privacy and automatically creates an
+ * anonymized copy of the table. Adapts templated query file if sensitive values are present.
+ *
+ * @param xmlConfig XML configuration
+ * @param configFile Configuration file path
+ * @throws Exception if anonymization fails
+ */
+ public void applyAnonymization(XMLConfiguration xmlConfig, String configFile) throws Exception {
+ try {
+ if (xmlConfig.configurationsAt("/anonymization/table").size() > 0) {
+ String templatesPath = "";
+ if (xmlConfig.containsKey("query_templates_file")) {
+ templatesPath = xmlConfig.getString("query_templates_file");
+ }
+
+ log.info("Starting the Anonymization process");
+ log.info(BenchmarkConstants.SINGLE_LINE);
+
+ String osCommand =
+ System.getProperty("os.name").startsWith("Windows") ? "python" : "python3";
+
+ ProcessBuilder processBuilder =
+ new ProcessBuilder(
+ osCommand, "scripts/anonymization/src/anonymizer.py", configFile, templatesPath);
+
+ // Redirect Output stream of the script to get live feedback
+ processBuilder.inheritIO();
+ Process process = processBuilder.start();
+ int exitCode = process.waitFor();
+
+ if (exitCode != 0) {
+ throw new Exception("Anonymization program exited with a non-zero status code");
+ }
+
+ log.info("Finished the Anonymization process for all tables");
+ log.info(BenchmarkConstants.SINGLE_LINE);
+ }
+ } catch (Exception e) {
+ log.error("Anonymization failed", e);
+ throw e;
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/execution/BenchmarkConfigurationFactory.java b/src/main/java/com/oltpbenchmark/execution/BenchmarkConfigurationFactory.java
new file mode 100644
index 0000000..748099e
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/execution/BenchmarkConfigurationFactory.java
@@ -0,0 +1,461 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.execution;
+
+import static com.oltpbenchmark.WorkloadConfiguration.UNINITIALIZED_TIME;
+
+import com.oltpbenchmark.Phase;
+import com.oltpbenchmark.WorkloadConfiguration;
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.api.TransactionType;
+import com.oltpbenchmark.types.DatabaseType;
+import com.oltpbenchmark.util.ClassUtil;
+import com.oltpbenchmark.util.ImmutableMonitorInfo;
+import com.oltpbenchmark.util.MonitorInfo;
+import com.oltpbenchmark.util.StringUtil;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.collections4.map.ListOrderedMap;
+import org.apache.commons.configuration2.HierarchicalConfiguration;
+import org.apache.commons.configuration2.XMLConfiguration;
+import org.apache.commons.configuration2.tree.ImmutableNode;
+
+/** Factory for creating and configuring benchmark modules. */
+@Slf4j
+@RequiredArgsConstructor
+public class BenchmarkConfigurationFactory {
+ private final TransactionTypeLoader txnTypeLoader;
+
+ /**
+ * Create benchmark modules from configuration
+ *
+ * @param targetList Array of benchmark names to create
+ * @param argsLine Command line arguments
+ * @param xmlConfig XML configuration
+ * @param pluginConfig Plugin configuration
+ * @param monitorInfo Monitoring configuration
+ * @return Result containing benchmark modules and active transaction types
+ * @throws Exception if creation fails
+ */
+ public FactoryResult createBenchmarks(
+ String[] targetList,
+ CommandLine argsLine,
+ XMLConfiguration xmlConfig,
+ XMLConfiguration pluginConfig,
+ MonitorInfo monitorInfo)
+ throws Exception {
+
+ List benchList = new ArrayList<>();
+ List activeTXTypes = new ArrayList<>();
+ int lastTxnId = 0;
+
+ for (String plugin : targetList) {
+
+ String pluginXpathString = "[@bench='" + plugin + "']";
+ int numTxnTypes =
+ xmlConfig
+ .configurationsAt("transactiontypes" + pluginXpathString + "/transactiontype")
+ .size();
+ if (numTxnTypes == 0 && targetList.length == 1) {
+ // if it is a single workload run, w/o attribute is used
+ pluginXpathString = "[not(@bench)]";
+ numTxnTypes =
+ xmlConfig
+ .configurationsAt("transactiontypes" + pluginXpathString + "/transactiontype")
+ .size();
+ }
+
+ BenchmarkCreationResult result =
+ createBenchmark(
+ plugin, pluginXpathString, argsLine, xmlConfig, pluginConfig, monitorInfo, lastTxnId);
+
+ benchList.add(result.benchmark);
+ activeTXTypes.addAll(result.activeTxTypes);
+ lastTxnId = result.lastTxnId;
+ }
+
+ return new FactoryResult(benchList, activeTXTypes);
+ }
+
+ /**
+ * Build monitoring configuration from command line
+ *
+ * @param argsLine Command line arguments
+ * @return MonitorInfo configuration
+ * @throws ParseException if parsing fails
+ */
+ public MonitorInfo buildMonitorInfo(CommandLine argsLine) throws ParseException {
+ ImmutableMonitorInfo.Builder builder = ImmutableMonitorInfo.builder();
+
+ if (argsLine.hasOption("im")) {
+ builder.monitoringInterval(Integer.parseInt(argsLine.getOptionValue("im")));
+ }
+
+ if (argsLine.hasOption("mt")) {
+ switch (argsLine.getOptionValue("mt")) {
+ case "advanced":
+ builder.monitoringType(MonitorInfo.MonitoringType.ADVANCED);
+ break;
+ case "throughput":
+ builder.monitoringType(MonitorInfo.MonitoringType.THROUGHPUT);
+ break;
+ default:
+ throw new ParseException(
+ "Monitoring type '"
+ + argsLine.getOptionValue("mt")
+ + "' is undefined, allowed values are: advanced/throughput");
+ }
+ }
+
+ return builder.build();
+ }
+
+ private BenchmarkCreationResult createBenchmark(
+ String plugin,
+ String pluginXpathString,
+ CommandLine argsLine,
+ XMLConfiguration xmlConfig,
+ XMLConfiguration pluginConfig,
+ MonitorInfo monitorInfo,
+ int lastTxnId)
+ throws Exception {
+
+ // Load workload configuration
+ WorkloadConfiguration wrkld =
+ ConfigurationLoader.loadWorkloadConfiguration(plugin, argsLine, xmlConfig);
+
+ // Set monitoring if enabled
+ setMonitoring(wrkld, monitorInfo, xmlConfig);
+
+ // Create benchmark module
+ BenchmarkModule bench = createBenchmarkModule(plugin, pluginConfig, wrkld);
+
+ // Log initialization
+ logBenchmarkInit(plugin, bench, wrkld);
+
+ // Load transaction types
+ TransactionTypeLoader.LoaderResult txnResult =
+ txnTypeLoader.loadTransactionTypes(xmlConfig, pluginXpathString, bench, lastTxnId);
+ wrkld.setTransTypes(txnResult.getTransactionTypes());
+
+ // Load transaction groupings
+ txnTypeLoader.loadTransactionGroupings(
+ xmlConfig,
+ pluginXpathString,
+ txnResult.getTransactionTypes().size() - 1); // -1 for INVALID type
+
+ // Set after load script if present
+ if (xmlConfig.containsKey("afterload")) {
+ bench.setAfterLoadScriptPath(xmlConfig.getString("afterload"));
+ }
+
+ // Load phases
+ loadPhases(wrkld, xmlConfig, plugin, argsLine);
+
+ // Validate phases
+ validatePhases(wrkld, txnResult.getTransactionTypes().size() - 1);
+
+ // Initialize workload
+ wrkld.init();
+
+ return new BenchmarkCreationResult(
+ bench, txnResult.getActiveTxTypes(), txnResult.getLastTxnId());
+ }
+
+ private void setMonitoring(
+ WorkloadConfiguration wrkld, MonitorInfo monitorInfo, XMLConfiguration xmlConfig) {
+ if (monitorInfo.getMonitoringInterval() > 0
+ && monitorInfo.getMonitoringType() == MonitorInfo.MonitoringType.ADVANCED
+ && DatabaseType.get(xmlConfig.getString("type")).shouldCreateMonitoringPrefix()) {
+ log.info("Advanced monitoring enabled, prefix will be added to queries.");
+ wrkld.setAdvancedMonitoringEnabled(true);
+ }
+ }
+
+ private BenchmarkModule createBenchmarkModule(
+ String plugin, XMLConfiguration pluginConfig, WorkloadConfiguration wrkld) throws Exception {
+
+ String classname = pluginConfig.getString("/plugin[@name='" + plugin + "']");
+
+ if (classname == null) {
+ throw new ParseException("Plugin " + plugin + " is undefined in config/plugin.xml");
+ }
+
+ return ClassUtil.newInstance(
+ classname, new Object[] {wrkld}, new Class[] {WorkloadConfiguration.class});
+ }
+
+ private void logBenchmarkInit(String plugin, BenchmarkModule bench, WorkloadConfiguration wrkld) {
+ Map initDebug = new ListOrderedMap<>();
+ initDebug.put(
+ "Benchmark", String.format("%s {%s}", plugin.toUpperCase(), bench.getClass().getName()));
+ initDebug.put("Type", wrkld.getDatabaseType());
+ initDebug.put("Driver", wrkld.getDriverClass());
+ initDebug.put("URL", wrkld.getUrl());
+ initDebug.put("Isolation", wrkld.getIsolationString());
+ initDebug.put("Batch Size", wrkld.getBatchSize());
+ initDebug.put("Scale Factor", wrkld.getScaleFactor());
+ initDebug.put("Terminals", wrkld.getTerminals());
+ initDebug.put("New Connection Per Txn", wrkld.isNewConnectionPerTxn());
+ initDebug.put("Reconnect on Connection Failure", wrkld.isReconnectOnConnectionFailure());
+
+ if (wrkld.getSelectivity() != -1) {
+ initDebug.put("Selectivity", wrkld.getSelectivity());
+ }
+
+ log.info("{}\n\n{}", BenchmarkConstants.SINGLE_LINE, StringUtil.formatMaps(initDebug));
+ log.info(BenchmarkConstants.SINGLE_LINE);
+ }
+
+ private void loadPhases(
+ WorkloadConfiguration wrkld,
+ XMLConfiguration xmlConfig,
+ String plugin,
+ CommandLine argsLine) {
+ String pluginXpathString = "[@bench='" + plugin + "']";
+ String[] targetList = argsLine.getOptionValue("b").split(",");
+ int terminals = wrkld.getTerminals();
+
+ int size = xmlConfig.configurationsAt("/works/work").size();
+ for (int i = 1; i < size + 1; i++) {
+ final HierarchicalConfiguration work =
+ xmlConfig.configurationAt("works/work[" + i + "]");
+
+ PhaseConfiguration phaseConfig =
+ loadPhaseConfiguration(work, pluginXpathString, targetList, terminals);
+
+ if (wrkld.getRunTimeInSeconds() != UNINITIALIZED_TIME) {
+ phaseConfig.time = wrkld.getRunTimeInSeconds();
+ }
+
+ wrkld.addPhase(
+ i,
+ phaseConfig.time,
+ phaseConfig.warmup,
+ phaseConfig.rate,
+ phaseConfig.weights,
+ phaseConfig.rateLimited,
+ phaseConfig.disabled,
+ phaseConfig.serial,
+ phaseConfig.timed,
+ phaseConfig.activeTerminals,
+ phaseConfig.arrival);
+ }
+ }
+
+ private PhaseConfiguration loadPhaseConfiguration(
+ HierarchicalConfiguration work,
+ String pluginTest,
+ String[] targetList,
+ int terminals) {
+
+ PhaseConfiguration config = new PhaseConfiguration();
+
+ // Load weights
+ List weight_strings;
+ if (targetList.length > 1 || work.containsKey("weights[@bench]")) {
+ weight_strings = Arrays.asList(work.getString("weights" + pluginTest).split("\\s*,\\s*"));
+ } else {
+ weight_strings = Arrays.asList(work.getString("weights[not(@bench)]").split("\\s*,\\s*"));
+ }
+
+ // Parse weights
+ config.weights = new ArrayList<>();
+ double totalWeight = 0;
+ for (String weightString : weight_strings) {
+ double weight = Double.parseDouble(weightString);
+ totalWeight += weight;
+ config.weights.add(weight);
+ }
+
+ long roundedWeight = Math.round(totalWeight);
+ if (roundedWeight != 100) {
+ log.warn(
+ "rounded weight [{}] does not equal 100. Original weight is [{}]",
+ roundedWeight,
+ totalWeight);
+ }
+
+ // Load rate configuration
+ loadRateConfiguration(work, pluginTest, config);
+
+ // Load other settings
+ config.arrival =
+ work.getString("@arrival", "regular").equalsIgnoreCase("POISSON")
+ ? Phase.Arrival.POISSON
+ : Phase.Arrival.REGULAR;
+ config.serial = Boolean.parseBoolean(work.getString("serial", Boolean.FALSE.toString()));
+
+ config.activeTerminals = work.getInt("active_terminals[not(@bench)]", terminals);
+ config.activeTerminals = work.getInt("active_terminals" + pluginTest, config.activeTerminals);
+
+ if (config.serial && config.activeTerminals != 1) {
+ log.warn("Serial ordering is enabled, so # of active terminals is clamped to 1.");
+ config.activeTerminals = 1;
+ }
+
+ if (config.activeTerminals > terminals) {
+ log.error(
+ "Configuration error: Number of active terminals is bigger than "
+ + "the total number of terminals");
+ System.exit(-1);
+ }
+
+ config.time = work.getInt("/time", 0);
+
+ config.warmup = work.getInt("/warmup", 0);
+ config.timed = (config.time > 0);
+
+ validatePhaseSettings(config);
+
+ return config;
+ }
+
+ private void loadRateConfiguration(
+ HierarchicalConfiguration work, String pluginTest, PhaseConfiguration config) {
+
+ String rateString = work.getString("rate[not(@bench)]", "");
+ rateString = work.getString("rate" + pluginTest, rateString);
+
+ parseRate(rateString, config);
+ }
+
+ private void parseRate(String rateString, PhaseConfiguration config) {
+ if (rateString.equals(BenchmarkConstants.RATE_DISABLED)) {
+ config.disabled = true;
+ config.rate = 1;
+ config.rateLimited = true;
+ } else if (rateString.equals(BenchmarkConstants.RATE_UNLIMITED)) {
+ config.rateLimited = false;
+ config.rate = 1;
+ } else if (rateString.isEmpty()) {
+ log.error("Please specify the rate for phase");
+ System.exit(-1);
+ } else {
+ try {
+ config.rate = Double.parseDouble(rateString);
+ config.rateLimited = true;
+ if (config.rate <= 0) {
+ log.error("Rate limit must be at least 0. Use unlimited or disabled values instead.");
+ System.exit(-1);
+ }
+ } catch (NumberFormatException e) {
+ log.error(
+ String.format(
+ "Rate string must be '%s', '%s' or a number",
+ BenchmarkConstants.RATE_DISABLED, BenchmarkConstants.RATE_UNLIMITED));
+ System.exit(-1);
+ }
+ }
+ }
+
+ private void validatePhaseSettings(PhaseConfiguration config) {
+ if (!config.timed) {
+ if (config.serial) {
+ log.info("Timer disabled for serial run; will execute all queries exactly once.");
+ } else {
+ log.error(
+ "Must provide positive time bound for non-serial executions. "
+ + "Either provide a valid time or enable serial mode.");
+ System.exit(-1);
+ }
+ } else if (config.serial) {
+ log.info(
+ "Timer enabled for serial run; will run queries serially "
+ + "in a loop until the timer expires.");
+ }
+
+ if (config.warmup < 0) {
+ log.error("Must provide non-negative time bound for warmup.");
+ System.exit(-1);
+ }
+ }
+
+ private void validatePhases(WorkloadConfiguration wrkld, int numTxnTypes) {
+ int j = 0;
+ for (Phase p : wrkld.getPhases()) {
+ j++;
+ if (p.getWeightCount() != numTxnTypes) {
+ log.error(
+ String.format(
+ "Configuration files is inconsistent, phase %d contains %d weights "
+ + "but you defined %d transaction types",
+ j, p.getWeightCount(), numTxnTypes));
+ if (p.isSerial()) {
+ log.error(
+ "However, note that since this a serial phase, the weights are "
+ + "irrelevant (but still must be included---sorry).");
+ }
+ System.exit(-1);
+ }
+ }
+ }
+
+ /** Result of benchmark creation */
+ private static class BenchmarkCreationResult {
+ final BenchmarkModule benchmark;
+ final List activeTxTypes;
+ final int lastTxnId;
+
+ BenchmarkCreationResult(
+ BenchmarkModule benchmark, List activeTxTypes, int lastTxnId) {
+ this.benchmark = benchmark;
+ this.activeTxTypes = activeTxTypes;
+ this.lastTxnId = lastTxnId;
+ }
+ }
+
+ /** Configuration for a phase */
+ private static class PhaseConfiguration {
+ double rate;
+ boolean rateLimited;
+ boolean disabled;
+ ArrayList weights;
+ Phase.Arrival arrival;
+ boolean serial;
+ int activeTerminals;
+ int time;
+ int warmup;
+ boolean timed;
+ }
+
+ /** Result of factory operations */
+ public static class FactoryResult {
+ private final List benchmarks;
+ private final List activeTxTypes;
+
+ public FactoryResult(List benchmarks, List activeTxTypes) {
+ this.benchmarks = benchmarks;
+ this.activeTxTypes = activeTxTypes;
+ }
+
+ public List getBenchmarks() {
+ return benchmarks;
+ }
+
+ public List getActiveTxTypes() {
+ return activeTxTypes;
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/execution/BenchmarkConstants.java b/src/main/java/com/oltpbenchmark/execution/BenchmarkConstants.java
new file mode 100644
index 0000000..fb0236a
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/execution/BenchmarkConstants.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.execution;
+
+import com.oltpbenchmark.util.StringUtil;
+
+/** Constants used throughout the benchmark execution. */
+public final class BenchmarkConstants {
+
+ // Formatting constants
+ public static final String SINGLE_LINE = StringUtil.repeat("=", 70);
+
+ // Rate constants
+ public static final String RATE_DISABLED = "disabled";
+ public static final String RATE_UNLIMITED = "unlimited";
+
+ // Default values
+ public static final String DEFAULT_OUTPUT_DIRECTORY = "results";
+ public static final int DEFAULT_WINDOW_SIZE = 5;
+ public static final double DEFAULT_CONNECTION_RATE = 10.0;
+ public static final int DEFAULT_BATCH_SIZE = 128;
+ public static final int DEFAULT_MAX_RETRIES = 3;
+
+ // Namespace defaults
+ public static final String DEFAULT_YCSB_NAMESPACE = "YcsbMetrics";
+ public static final String DEFAULT_TPCC_NAMESPACE = "TpccMetrics";
+ public static final String DEFAULT_YCSB_TESTNAME = "YcsbDefault";
+ public static final String DEFAULT_TPCC_TESTNAME = "TpccDefault";
+
+ // Private constructor to prevent instantiation
+ private BenchmarkConstants() {
+ throw new AssertionError("Cannot instantiate constants class");
+ }
+
+ /**
+ * Get default CloudWatch namespace for a benchmark
+ *
+ * @param benchmarkName Name of the benchmark
+ * @return Default namespace
+ */
+ public static String getDefaultNamespace(String benchmarkName) {
+ switch (benchmarkName) {
+ case "ycsb":
+ return DEFAULT_YCSB_NAMESPACE;
+ case "tpcc":
+ default:
+ return DEFAULT_TPCC_NAMESPACE;
+ }
+ }
+
+ /**
+ * Get default test name for a benchmark
+ *
+ * @param benchmarkName Name of the benchmark
+ * @return Default test name
+ */
+ public static String getDefaultTestName(String benchmarkName) {
+ switch (benchmarkName) {
+ case "ycsb":
+ return DEFAULT_YCSB_TESTNAME;
+ case "tpcc":
+ default:
+ return DEFAULT_TPCC_TESTNAME;
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/execution/BenchmarkOrchestrator.java b/src/main/java/com/oltpbenchmark/execution/BenchmarkOrchestrator.java
new file mode 100644
index 0000000..826ebdb
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/execution/BenchmarkOrchestrator.java
@@ -0,0 +1,224 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.execution;
+
+import com.oltpbenchmark.Results;
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.api.TransactionType;
+import com.oltpbenchmark.util.MonitorInfo;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.configuration2.XMLConfiguration;
+
+/** Orchestrates the benchmark execution workflow. */
+@Slf4j
+public class BenchmarkOrchestrator {
+
+ private final CommandLineHandler commandLineHandler;
+ private final BenchmarkConfigurationFactory benchmarkFactory;
+ private final DatabaseOperationsManager dbOpsManager;
+ private final WorkloadExecutor workloadExecutor;
+ private final ResultsManager resultsManager;
+ private final AnonymizationHandler anonymizationHandler;
+
+ public BenchmarkOrchestrator() throws Exception {
+ // Load plugin configuration first
+ XMLConfiguration pluginConfig = ConfigurationLoader.loadXMLConfiguration("config/plugin.xml");
+
+ // Initialize components
+ this.commandLineHandler = new CommandLineHandler(pluginConfig);
+ this.benchmarkFactory = new BenchmarkConfigurationFactory(new TransactionTypeLoader());
+ this.dbOpsManager = new DatabaseOperationsManager();
+ this.workloadExecutor = new WorkloadExecutor();
+ this.resultsManager = new ResultsManager();
+ this.anonymizationHandler = new AnonymizationHandler();
+ }
+
+ /**
+ * Run the benchmark based on command line arguments
+ *
+ * @param args Command line arguments
+ * @throws Exception if execution fails
+ */
+ public void run(String[] args) throws Exception {
+ // Parse command line
+ CommandLine argsLine = commandLineHandler.parse(args);
+
+ // Handle help
+ if (commandLineHandler.isHelpRequested(argsLine)) {
+ commandLineHandler.printUsage();
+ return;
+ }
+
+ // Validate required arguments
+ if (!commandLineHandler.validateRequiredArguments(argsLine)) {
+ commandLineHandler.printUsage();
+ return;
+ }
+
+ // Load configurations
+ String configFile = argsLine.getOptionValue("c");
+ XMLConfiguration xmlConfig = ConfigurationLoader.loadXMLConfiguration(configFile);
+ XMLConfiguration pluginConfig = ConfigurationLoader.loadXMLConfiguration("config/plugin.xml");
+
+ // Build monitoring configuration
+ MonitorInfo monitorInfo = benchmarkFactory.buildMonitorInfo(argsLine);
+
+ // Get target benchmarks
+ String targetBenchmarks = argsLine.getOptionValue("b");
+ String[] targetList = targetBenchmarks.split(",");
+
+ // Create benchmarks
+ BenchmarkConfigurationFactory.FactoryResult factoryResult =
+ benchmarkFactory.createBenchmarks(
+ targetList, argsLine, xmlConfig, pluginConfig, monitorInfo);
+
+ List benchmarks = factoryResult.getBenchmarks();
+ List activeTxTypes = factoryResult.getActiveTxTypes();
+
+ // Handle dialect export if requested
+ if (CommandLineHandler.isBooleanOptionSet(argsLine, "dialects-export")) {
+ handleDialectExport(benchmarks.get(0));
+ return;
+ }
+
+ // Execute database operations
+ executeDatabaseOperations(argsLine, benchmarks);
+
+ // Refresh catalogs
+ dbOpsManager.refreshCatalogs(benchmarks);
+
+ // Handle anonymization if requested
+ if (CommandLineHandler.isBooleanOptionSet(argsLine, "anonymize")) {
+ anonymizationHandler.applyAnonymization(xmlConfig, configFile);
+ }
+
+ // Execute workload if requested
+ if (CommandLineHandler.isBooleanOptionSet(argsLine, "execute")) {
+ executeWorkload(benchmarks, activeTxTypes, argsLine, xmlConfig, monitorInfo);
+ } else {
+ log.info("Skipping benchmark workload execution");
+ }
+ }
+
+ private void executeDatabaseOperations(CommandLine argsLine, List benchmarks)
+ throws SQLException, IOException, InterruptedException {
+
+ // Create databases
+ if (CommandLineHandler.isBooleanOptionSet(argsLine, "create")) {
+ try {
+ dbOpsManager.createDatabases(benchmarks);
+ // Refresh catalog after creation
+ } catch (Throwable ex) {
+ log.error("Unexpected error when creating benchmark database tables.", ex);
+ System.exit(1);
+ }
+ } else {
+ log.debug("Skipping creating benchmark database tables");
+ }
+ dbOpsManager.refreshCatalogs(benchmarks);
+ // Clear databases
+ if (CommandLineHandler.isBooleanOptionSet(argsLine, "clear")) {
+ try {
+ dbOpsManager.clearDatabases(benchmarks);
+ } catch (Throwable ex) {
+ log.error("Unexpected error when clearing benchmark database tables.", ex);
+ System.exit(1);
+ }
+ } else {
+ log.debug("Skipping clearing benchmark database tables");
+ }
+
+ // Load data
+ if (CommandLineHandler.isBooleanOptionSet(argsLine, "load")) {
+ try {
+ dbOpsManager.loadData(benchmarks);
+ } catch (Throwable ex) {
+ log.error("Unexpected error when loading benchmark database records.", ex);
+ System.exit(1);
+ }
+ } else {
+ log.debug("Skipping loading benchmark database records");
+ }
+ }
+
+ private void executeWorkload(
+ List benchmarks,
+ List activeTxTypes,
+ CommandLine argsLine,
+ XMLConfiguration xmlConfig,
+ MonitorInfo monitorInfo)
+ throws Exception {
+
+ try {
+ // Execute workload
+ Results results = workloadExecutor.execute(benchmarks, monitorInfo);
+
+ // Write histograms to console
+ resultsManager.writeHistograms(results);
+
+ // Build results configuration
+ ResultsConfiguration resultsConfig =
+ buildResultsConfiguration(argsLine, xmlConfig, activeTxTypes);
+
+ // Write results to files
+ resultsManager.writeResults(results, resultsConfig);
+
+ } catch (Throwable ex) {
+ log.error("Unexpected error when executing benchmarks.", ex);
+ System.exit(1);
+ }
+ }
+
+ private ResultsConfiguration buildResultsConfiguration(
+ CommandLine argsLine, XMLConfiguration xmlConfig, List activeTxTypes) {
+
+ String outputDirectory =
+ argsLine.hasOption("d")
+ ? argsLine.getOptionValue("d")
+ : BenchmarkConstants.DEFAULT_OUTPUT_DIRECTORY;
+
+ int windowSize =
+ Integer.parseInt(
+ argsLine.getOptionValue("s", String.valueOf(BenchmarkConstants.DEFAULT_WINDOW_SIZE)));
+
+ String name = String.join("-", argsLine.getOptionValue("b").split(","));
+ String baseFileName = name + "_" + com.oltpbenchmark.util.TimeUtil.getCurrentTimeString();
+
+ return new ResultsConfiguration(
+ outputDirectory, baseFileName, windowSize, activeTxTypes, argsLine, xmlConfig);
+ }
+
+ private void handleDialectExport(BenchmarkModule bench) {
+ if (bench.getStatementDialects() != null) {
+ log.info("Exporting StatementDialects for {}", bench);
+ String xml =
+ bench
+ .getStatementDialects()
+ .export(
+ bench.getWorkloadConfiguration().getDatabaseType(),
+ bench.getProcedures().values());
+ log.debug(xml);
+ System.exit(0);
+ }
+ throw new RuntimeException("No StatementDialects is available for " + bench);
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/execution/CommandLineHandler.java b/src/main/java/com/oltpbenchmark/execution/CommandLineHandler.java
new file mode 100644
index 0000000..8d8181c
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/execution/CommandLineHandler.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.execution;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.cli.*;
+import org.apache.commons.configuration2.XMLConfiguration;
+
+/** Handles command-line argument parsing and options building for the benchmark. */
+@Slf4j
+public class CommandLineHandler {
+ private final Options options;
+
+ public CommandLineHandler(XMLConfiguration pluginConfig) {
+ this.options = buildOptions(pluginConfig);
+ }
+
+ /**
+ * Parse command line arguments
+ *
+ * @param args Command line arguments
+ * @return CommandLine object with parsed arguments
+ * @throws ParseException if parsing fails
+ */
+ public CommandLine parse(String[] args) throws ParseException {
+ CommandLineParser parser = new DefaultParser();
+ return parser.parse(options, args);
+ }
+
+ /**
+ * Check if help was requested
+ *
+ * @param commandLine Parsed command line
+ * @return true if help was requested
+ */
+ public boolean isHelpRequested(CommandLine commandLine) {
+ return commandLine.hasOption("h");
+ }
+
+ /**
+ * Validate required arguments
+ *
+ * @param commandLine Parsed command line
+ * @return true if all required arguments are present
+ */
+ public boolean validateRequiredArguments(CommandLine commandLine) {
+ if (!commandLine.hasOption("c")) {
+ log.error("Missing Configuration file");
+ return false;
+ }
+ if (!commandLine.hasOption("b")) {
+ log.error("Missing Benchmark Class to load");
+ return false;
+ }
+ return true;
+ }
+
+ /** Print usage information */
+ public void printUsage() {
+ HelpFormatter hlpfrmt = new HelpFormatter();
+ hlpfrmt.printHelp("benchbase", options);
+ }
+
+ /**
+ * Check if a boolean option is set to true
+ *
+ * @param commandLine Parsed command line
+ * @param key Option key
+ * @return true if the option is set to true
+ */
+ public static boolean isBooleanOptionSet(CommandLine commandLine, String key) {
+ if (commandLine.hasOption(key)) {
+ log.debug("CommandLine has option '{}'. Checking whether set to true", key);
+ String val = commandLine.getOptionValue(key);
+ log.debug(String.format("CommandLine %s => %s", key, val));
+ return (val != null && val.equalsIgnoreCase("true"));
+ }
+ return false;
+ }
+
+ private Options buildOptions(XMLConfiguration pluginConfig) {
+ Options options = new Options();
+
+ // Core options
+ options.addOption(
+ "b",
+ "bench",
+ true,
+ "[required] Benchmark class. Currently supported: "
+ + pluginConfig.getList("/plugin//@name"));
+ options.addOption("c", "config", true, "[required] Workload configuration file");
+
+ // Database operations
+ options.addOption(null, "create", true, "Initialize the database for this benchmark");
+ options.addOption(null, "clear", true, "Clear all records in the database for this benchmark");
+ options.addOption(null, "load", true, "Load data using the benchmark's data loader");
+ options.addOption(
+ null, "anonymize", true, "Anonymize specified datasets using differential privacy");
+ options.addOption(null, "execute", true, "Execute the benchmark workload");
+
+ // Help and output options
+ options.addOption("h", "help", false, "Print this help");
+ options.addOption("s", "sample", true, "Sampling window");
+ options.addOption("im", "interval-monitor", true, "Monitoring Interval in milliseconds");
+ options.addOption("mt", "monitor-type", true, "Type of Monitoring (throughput/advanced)");
+ options.addOption(
+ "d",
+ "directory",
+ true,
+ "Base directory for the result files, default is current directory");
+ options.addOption(null, "dialects-export", true, "Export benchmark SQL to a dialects file");
+ options.addOption("jh", "json-histograms", true, "Export histograms to JSON file");
+
+ // Database configuration
+ options.addOption(null, "type", true, "Type of database. For e.g. POSTGRES");
+ options.addOption(null, "driver", true, "Driver class name. For e.g. org.postgresql.Driver");
+ options.addOption(null, "url", true, "URL to connect with the database");
+ options.addOption(null, "reconnectOnConnectionFailure", false, "Should reconnect on failure?");
+ options.addOption(
+ null, "isolation", true, "Isolation level. For e.g. TRANSACTION_SERIALIZABLE");
+ options.addOption(null, "batchsize", true, "Batch size. For e.g. 128");
+ options.addOption(null, "username", true, "Username to connect with the database.");
+ options.addOption(
+ null,
+ "scalefactor",
+ true,
+ "For TPCC, Number of warehouses. For e.g. 100. For YCSB, Scalefactor is *1000 the number of rows in the USERTABLE");
+
+ // Monitoring and performance options
+ options.addOption(
+ null,
+ "benchmarkTestName",
+ true,
+ "Benchmark test name. CW metrics are published under this name.");
+ options.addOption(null, "publishToCloudWatch", false, "Publish to CloudWatch.");
+ options.addOption(null, "region", true, "AWS region.");
+ options.addOption(null, "retries", true, "Number of retries.");
+ options.addOption(null, "loaderThreads", true, "Number of loader threads.");
+
+ // TPCC specific options
+ options.addOption(null, "skipIndexBuild", false, "Skip index build.");
+ options.addOption(null, "skipItemLoad", false, "Skip item load.");
+ options.addOption(null, "skipMainDataLoad", false, "Skip main data load.");
+ options.addOption(null, "terminals", true, "Number of terminals.");
+ options.addOption(null, "time", true, "Number of seconds to run the execution phase");
+ options.addOption(null, "stride", true, "Stride to cover warehouses. For e.g. 1");
+ options.addOption(
+ null,
+ "startWarehouseIndex",
+ true,
+ "Start warehouse index for TPCC (1 based index). For e.g. 1");
+ options.addOption(
+ null,
+ "endWarehouseIndex",
+ true,
+ "End warehouse index for TPCC (1 based index). For e.g. 100");
+
+ return options;
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/execution/ConfigurationLoader.java b/src/main/java/com/oltpbenchmark/execution/ConfigurationLoader.java
new file mode 100644
index 0000000..c70fdfb
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/execution/ConfigurationLoader.java
@@ -0,0 +1,172 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.execution;
+
+import static com.oltpbenchmark.WorkloadConfiguration.UNINITIALIZED_TIME;
+
+import com.oltpbenchmark.WorkloadConfiguration;
+import com.oltpbenchmark.types.DatabaseType;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.configuration2.XMLConfiguration;
+import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder;
+import org.apache.commons.configuration2.builder.fluent.Parameters;
+import org.apache.commons.configuration2.convert.DisabledListDelimiterHandler;
+import org.apache.commons.configuration2.ex.ConfigurationException;
+import org.apache.commons.configuration2.tree.xpath.XPathExpressionEngine;
+
+/** Handles loading and parsing of configuration files. */
+@Slf4j
+public class ConfigurationLoader {
+ /**
+ * Build an XMLConfiguration from a file
+ *
+ * @param filename Path to the configuration file
+ * @return XMLConfiguration object
+ * @throws ConfigurationException if the file cannot be loaded or parsed
+ */
+ public static XMLConfiguration loadXMLConfiguration(String filename)
+ throws ConfigurationException {
+ Parameters params = new Parameters();
+ FileBasedConfigurationBuilder builder =
+ new FileBasedConfigurationBuilder<>(XMLConfiguration.class)
+ .configure(
+ params
+ .xml()
+ .setFileName(filename)
+ .setListDelimiterHandler(new DisabledListDelimiterHandler())
+ .setExpressionEngine(new XPathExpressionEngine()));
+ return builder.getConfiguration();
+ }
+
+ /**
+ * Load workload configuration for a specific plugin
+ *
+ * @param plugin The plugin/benchmark name
+ * @param argsLine Command line arguments
+ * @param xmlConfig XML configuration
+ * @return Configured WorkloadConfiguration object
+ */
+ public static WorkloadConfiguration loadWorkloadConfiguration(
+ String plugin, CommandLine argsLine, XMLConfiguration xmlConfig) {
+
+ WorkloadConfiguration wrkld = new WorkloadConfiguration();
+ wrkld.setBenchmarkName(plugin);
+ wrkld.setXmlConfig(xmlConfig);
+
+ OptionsRetriever optionsRetriever = new OptionsRetriever(argsLine, xmlConfig);
+
+ configureDatabaseSettings(wrkld, optionsRetriever);
+ configureBenchmarkSettings(wrkld, optionsRetriever);
+ configureBenchmarkLoaderSettings(wrkld, optionsRetriever);
+ configureMetricSettings(wrkld, optionsRetriever);
+ configureTerminals(wrkld, argsLine, xmlConfig, plugin);
+ configureOtherSettings(wrkld, optionsRetriever, xmlConfig);
+
+ return wrkld;
+ }
+
+ private static void configureDatabaseSettings(
+ WorkloadConfiguration wrkld, OptionsRetriever optionsRetriever) {
+ wrkld.setDatabaseType(DatabaseType.get(optionsRetriever.getString("type")));
+ wrkld.setDriverClass(optionsRetriever.getString("driver"));
+ wrkld.setUrl(optionsRetriever.getString("url"));
+ wrkld.setUsername(optionsRetriever.getString("username"));
+ wrkld.setPassword(optionsRetriever.getString("password"));
+ wrkld.setNewConnectionPerTxn(optionsRetriever.getBoolean("newConnectionPerTxn", false));
+ wrkld.setReconnectOnConnectionFailure(
+ optionsRetriever.getBoolean("reconnectOnConnectionFailure", false));
+ }
+
+ private static void configureBenchmarkSettings(
+ WorkloadConfiguration wrkld, OptionsRetriever optionsRetriever) {
+ wrkld.setConnectionRate(
+ optionsRetriever.getDouble("connectionRate", BenchmarkConstants.DEFAULT_CONNECTION_RATE));
+ wrkld.setRandomSeed(optionsRetriever.getInt("randomSeed", -1));
+ wrkld.setBatchSize(optionsRetriever.getInt("batchsize", BenchmarkConstants.DEFAULT_BATCH_SIZE));
+ wrkld.setMaxRetries(optionsRetriever.getInt("retries", BenchmarkConstants.DEFAULT_MAX_RETRIES));
+ wrkld.setDataDir(optionsRetriever.getString("datadir", "."));
+ wrkld.setDDLPath(optionsRetriever.getString("ddlpath", null));
+
+ wrkld.setScaleFactor(optionsRetriever.getDouble("scalefactor", 1.0));
+
+ wrkld.setStride(optionsRetriever.getInt("stride", 1));
+ wrkld.setStartWarehouseIndex(optionsRetriever.getInt("startWarehouseIndex", 1));
+ wrkld.setEndWarehouseIndex(
+ optionsRetriever.getInt("endWarehouseIndex", (int) wrkld.getScaleFactor()));
+
+ wrkld.setRunTimeInSeconds(optionsRetriever.getInt("time", UNINITIALIZED_TIME));
+ }
+
+ private static void configureBenchmarkLoaderSettings(
+ WorkloadConfiguration wrkld, OptionsRetriever optionsRetriever) {
+ wrkld.setLoaderThreads(optionsRetriever.getInt("loaderThreads", wrkld.getLoaderThreads()));
+
+ wrkld.setSkipIndexBuild(optionsRetriever.getBooleanWithoutArg("skipIndexBuild", true));
+ wrkld.setSkipItemLoad(optionsRetriever.getBooleanWithoutArg("skipItemLoad", false));
+ wrkld.setSkipMainDataLoad(optionsRetriever.getBooleanWithoutArg("skipMainDataLoad", false));
+ }
+
+ private static void configureMetricSettings(
+ WorkloadConfiguration wrkld, OptionsRetriever optionsRetriever) {
+ wrkld.setDisableLocalMetrics(optionsRetriever.getBoolean("disableLocalMetrics", false));
+
+ wrkld.setPublishToCloudWatch(optionsRetriever.getBoolean("publishToCloudWatch", false));
+ wrkld.setBenchmarkTestName(
+ optionsRetriever.getString(
+ "benchmarkTestName", BenchmarkConstants.getDefaultTestName(wrkld.getBenchmarkName())));
+ wrkld.setNamespace(
+ optionsRetriever.getString(
+ "namespace", BenchmarkConstants.getDefaultNamespace(wrkld.getBenchmarkName())));
+ wrkld.setRegion(optionsRetriever.getString("region", null));
+ }
+
+ private static void configureTerminals(
+ WorkloadConfiguration wrkld,
+ CommandLine argsLine,
+ XMLConfiguration xmlConfig,
+ String plugin) {
+ String pluginXpathString = "[@bench='" + plugin + "']";
+ int terminals;
+
+ if (argsLine.hasOption("terminals")) {
+ terminals = Integer.parseInt(argsLine.getOptionValue("terminals"));
+ } else {
+ terminals = xmlConfig.getInt("terminals[not(@bench)]", 0);
+ terminals = xmlConfig.getInt("terminals" + pluginXpathString, terminals);
+ }
+ wrkld.setTerminals(terminals);
+ }
+
+ private static void configureOtherSettings(
+ WorkloadConfiguration wrkld, OptionsRetriever optionsRetriever, XMLConfiguration xmlConfig) {
+ String pluginXpathString = "[@bench='" + wrkld.getBenchmarkName() + "']";
+ String isolationMode =
+ xmlConfig.getString("isolation[not(@bench)]", "TRANSACTION_SERIALIZABLE");
+ wrkld.setIsolationMode(
+ optionsRetriever.getString("isolation" + pluginXpathString, isolationMode));
+
+ // Set selectivity if available
+ try {
+ double selectivity = xmlConfig.getDouble("selectivity");
+ wrkld.setSelectivity(selectivity);
+ } catch (Exception e) {
+ // Selectivity is optional, so we ignore if not present
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/execution/DatabaseOperationsManager.java b/src/main/java/com/oltpbenchmark/execution/DatabaseOperationsManager.java
new file mode 100644
index 0000000..baa4518
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/execution/DatabaseOperationsManager.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.execution;
+
+import com.oltpbenchmark.api.BenchmarkModule;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+
+/** Manages database operations for benchmarks including creating, clearing, and loading data. */
+@Slf4j
+public class DatabaseOperationsManager {
+ /**
+ * Create database tables for all benchmarks
+ *
+ * @param benchmarks List of benchmark modules
+ * @throws SQLException if database operation fails
+ * @throws IOException if I/O operation fails
+ */
+ public void createDatabases(List benchmarks) throws SQLException, IOException {
+ for (BenchmarkModule benchmark : benchmarks) {
+ log.info("Creating new {} database...", benchmark.getBenchmarkName().toUpperCase());
+ createDatabase(benchmark);
+ log.info("Finished creating new {} database...", benchmark.getBenchmarkName().toUpperCase());
+ }
+ }
+
+ /**
+ * Clear database tables for all benchmarks
+ *
+ * @param benchmarks List of benchmark modules
+ * @throws SQLException if database operation fails
+ */
+ public void clearDatabases(List benchmarks) throws SQLException {
+ for (BenchmarkModule benchmark : benchmarks) {
+ log.info("Clearing {} database...", benchmark.getBenchmarkName().toUpperCase());
+ benchmark.refreshCatalog();
+ benchmark.clearDatabase();
+ benchmark.refreshCatalog();
+ log.info("Finished clearing {} database...", benchmark.getBenchmarkName().toUpperCase());
+ }
+ }
+
+ /**
+ * Load data into all benchmarks
+ *
+ * @param benchmarks List of benchmark modules
+ * @throws IOException if I/O operation fails
+ * @throws SQLException if database operation fails
+ * @throws InterruptedException if loading is interrupted
+ */
+ public void loadData(List benchmarks)
+ throws IOException, SQLException, InterruptedException {
+ for (BenchmarkModule benchmark : benchmarks) {
+ log.info("Loading data into {} database...", benchmark.getBenchmarkName().toUpperCase());
+ loadDatabase(benchmark);
+ log.info(
+ "Finished loading data into {} database...", benchmark.getBenchmarkName().toUpperCase());
+ }
+ }
+
+ /**
+ * Refresh catalogs for all benchmarks
+ *
+ * @param benchmarks List of benchmark modules
+ * @throws SQLException if database operation fails
+ */
+ public void refreshCatalogs(List benchmarks) throws SQLException {
+ for (BenchmarkModule benchmark : benchmarks) {
+ benchmark.refreshCatalog();
+ }
+ }
+
+ /**
+ * Create database for a single benchmark
+ *
+ * @param bench Benchmark module
+ * @throws SQLException if database operation fails
+ * @throws IOException if I/O operation fails
+ */
+ private void createDatabase(BenchmarkModule bench) throws SQLException, IOException {
+ log.debug(String.format("Creating %s Database", bench));
+ bench.createDatabase();
+ }
+
+ /**
+ * Load database for a single benchmark
+ *
+ * @param bench Benchmark module
+ * @throws IOException if I/O operation fails
+ * @throws SQLException if database operation fails
+ * @throws InterruptedException if loading is interrupted
+ */
+ private void loadDatabase(BenchmarkModule bench)
+ throws IOException, SQLException, InterruptedException {
+ log.debug(String.format("Loading %s Database", bench));
+ bench.loadDatabase();
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/execution/OptionsRetriever.java b/src/main/java/com/oltpbenchmark/execution/OptionsRetriever.java
new file mode 100644
index 0000000..e340f00
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/execution/OptionsRetriever.java
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.execution;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.configuration2.XMLConfiguration;
+
+/**
+ * Helper class to retrieve options from either command line arguments or XML configuration. Command
+ * line arguments take precedence over XML configuration values.
+ */
+public class OptionsRetriever {
+
+ private final CommandLine argsLine;
+ private final XMLConfiguration xmlConfig;
+
+ public OptionsRetriever(CommandLine argsLine, XMLConfiguration xmlConfig) {
+ this.argsLine = argsLine;
+ this.xmlConfig = xmlConfig;
+ }
+
+ /**
+ * Get a string value, preferring command line over XML config
+ *
+ * @param key The option key
+ * @return The value or null if not found
+ */
+ public String getString(String key) {
+ if (argsLine.hasOption(key)) {
+ return argsLine.getOptionValue(key);
+ }
+ return xmlConfig.getString(key);
+ }
+
+ /**
+ * Get a string value with a default
+ *
+ * @param key The option key
+ * @param defaultValue The default value if not found
+ * @return The value or default
+ */
+ public String getString(String key, String defaultValue) {
+ if (argsLine.hasOption(key)) {
+ return argsLine.getOptionValue(key);
+ }
+ return xmlConfig.getString(key, defaultValue);
+ }
+
+ /**
+ * Get an integer value, preferring command line over XML config
+ *
+ * @param key The option key
+ * @return The integer value
+ * @throws NumberFormatException if the value cannot be parsed as an integer
+ */
+ public int getInt(String key) {
+ if (argsLine.hasOption(key)) {
+ return Integer.parseInt(argsLine.getOptionValue(key));
+ }
+ return xmlConfig.getInt(key);
+ }
+
+ /**
+ * Get an integer value with a default
+ *
+ * @param key The option key
+ * @param defaultValue The default value if not found
+ * @return The integer value or default
+ */
+ public int getInt(String key, int defaultValue) {
+ if (argsLine.hasOption(key)) {
+ return Integer.parseInt(argsLine.getOptionValue(key));
+ }
+ return xmlConfig.getInt(key, defaultValue);
+ }
+
+ /**
+ * Get a double value, preferring command line over XML config
+ *
+ * @param key The option key
+ * @return The double value
+ * @throws NumberFormatException if the value cannot be parsed as a double
+ */
+ public double getDouble(String key) {
+ if (argsLine.hasOption(key)) {
+ return Double.parseDouble(argsLine.getOptionValue(key));
+ }
+ return xmlConfig.getDouble(key);
+ }
+
+ /**
+ * Get a double value with a default
+ *
+ * @param key The option key
+ * @param defaultValue The default value if not found
+ * @return The double value or default
+ */
+ public double getDouble(String key, double defaultValue) {
+ if (argsLine.hasOption(key)) {
+ return Double.parseDouble(argsLine.getOptionValue(key));
+ }
+ return xmlConfig.getDouble(key, defaultValue);
+ }
+
+ /**
+ * Get a boolean value with a default
+ *
+ * @param key The option key
+ * @param defaultValue The default value if not found
+ * @return The boolean value or default
+ */
+ public boolean getBoolean(String key, boolean defaultValue) {
+ if (argsLine.hasOption(key)) {
+ return Boolean.parseBoolean(argsLine.getOptionValue(key));
+ }
+ return xmlConfig.getBoolean(key, defaultValue);
+ }
+
+ /**
+ * Return boolean based on the presence of option (ignoring the option value).
+ *
+ * @param key The option key
+ * @param defaultValue The default value if not found
+ * @return The boolean value or default
+ */
+ public boolean getBooleanWithoutArg(String key, boolean defaultValue) {
+ if (argsLine.hasOption(key)) {
+ return true;
+ }
+ return xmlConfig.getBoolean(key, defaultValue);
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/execution/ResultsConfiguration.java b/src/main/java/com/oltpbenchmark/execution/ResultsConfiguration.java
new file mode 100644
index 0000000..95cc396
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/execution/ResultsConfiguration.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.execution;
+
+import com.oltpbenchmark.api.TransactionType;
+import java.util.List;
+import lombok.Value;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.configuration2.XMLConfiguration;
+
+/** Configuration for results output. */
+@Value
+public class ResultsConfiguration {
+
+ String outputDirectory;
+ String baseFileName;
+ int windowSize;
+ List activeTxTypes;
+ CommandLine commandLine;
+ XMLConfiguration xmlConfig;
+
+ /**
+ * Check if JSON histograms should be written
+ *
+ * @return true if JSON histograms should be written
+ */
+ public boolean shouldWriteJsonHistograms() {
+ return commandLine.hasOption("json-histograms");
+ }
+
+ /**
+ * Get the JSON histograms filename
+ *
+ * @return JSON histograms filename
+ */
+ public String getJsonHistogramsFileName() {
+ return commandLine.getOptionValue("json-histograms");
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/execution/ResultsManager.java b/src/main/java/com/oltpbenchmark/execution/ResultsManager.java
new file mode 100644
index 0000000..3130281
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/execution/ResultsManager.java
@@ -0,0 +1,249 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.execution;
+
+import com.oltpbenchmark.Results;
+import com.oltpbenchmark.api.TransactionType;
+import com.oltpbenchmark.types.State;
+import com.oltpbenchmark.util.FileUtil;
+import com.oltpbenchmark.util.JSONSerializable;
+import com.oltpbenchmark.util.JSONUtil;
+import com.oltpbenchmark.util.ResultWriter;
+import com.oltpbenchmark.util.StringUtil;
+import com.oltpbenchmark.util.TimeUtil;
+import java.io.File;
+import java.io.PrintStream;
+import java.util.HashMap;
+import java.util.Map;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+/** Manages writing benchmark results to various output formats. */
+@Slf4j
+public class ResultsManager {
+
+ /**
+ * Write all results to files
+ *
+ * @param results Benchmark results
+ * @param config Results configuration
+ * @throws Exception if writing fails
+ */
+ public void writeResults(Results results, ResultsConfiguration config) throws Exception {
+
+ // Ensure output directory exists
+ FileUtil.makeDirIfNotExists(config.getOutputDirectory());
+
+ // Create result writer
+ ResultWriter rw = new ResultWriter(results, config.getXmlConfig(), config.getCommandLine());
+
+ // Generate base filename
+ String name =
+ StringUtils.join(StringUtils.split(config.getCommandLine().getOptionValue("b"), ','), '-');
+ String baseFileName = name + "_" + TimeUtil.getCurrentTimeString();
+
+ // Write raw results
+ writeRawResults(rw, config, baseFileName);
+
+ // Write samples
+ writeSamples(rw, config, baseFileName);
+
+ // Write summary
+ writeSummary(rw, config, baseFileName);
+
+ // Write parameters
+ writeParams(rw, config, baseFileName);
+
+ // Write metrics if available
+ if (rw.hasMetrics()) {
+ writeMetrics(rw, config, baseFileName);
+ }
+
+ // Write configuration
+ writeConfig(rw, config, baseFileName);
+
+ // Write results CSV
+ writeResultsCsv(rw, config, baseFileName);
+
+ // Write per-transaction results
+ writePerTransactionResults(rw, config, baseFileName);
+
+ // Handle JSON histograms if requested
+ if (config.shouldWriteJsonHistograms()) {
+ String histogramJson = writeJSONHistograms(results);
+ FileUtil.writeStringToFile(new File(config.getJsonHistogramsFileName()), histogramJson);
+ log.info("Histograms JSON Data: " + config.getJsonHistogramsFileName());
+ }
+
+ // Check for errors
+ if (results.getState() == State.ERROR) {
+ throw new RuntimeException(
+ "Errors encountered during benchmark execution. See output above for details.");
+ }
+ }
+
+ /**
+ * Write histograms to console
+ *
+ * @param results Benchmark results
+ */
+ public void writeHistograms(Results results) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n");
+
+ sb.append(StringUtil.bold("Completed Transactions:"))
+ .append("\n")
+ .append(results.getSuccess())
+ .append("\n\n");
+
+ sb.append(StringUtil.bold("Aborted Transactions:"))
+ .append("\n")
+ .append(results.getAbort())
+ .append("\n\n");
+
+ sb.append(StringUtil.bold("Rejected Transactions (Server Retry):"))
+ .append("\n")
+ .append(results.getRetry())
+ .append("\n\n");
+
+ sb.append(StringUtil.bold("Rejected Transactions (Retry Different):"))
+ .append("\n")
+ .append(results.getRetryDifferent())
+ .append("\n\n");
+
+ sb.append(StringUtil.bold("Unexpected SQL Errors:"))
+ .append("\n")
+ .append(results.getError())
+ .append("\n\n");
+
+ sb.append(StringUtil.bold("Unknown Status Transactions:"))
+ .append("\n")
+ .append(results.getUnknown())
+ .append("\n\n");
+
+ if (!results.getAbortMessages().isEmpty()) {
+ sb.append("\n\n")
+ .append(StringUtil.bold("User Aborts:"))
+ .append("\n")
+ .append(results.getAbortMessages());
+ }
+
+ log.info(BenchmarkConstants.SINGLE_LINE);
+ log.info("Workload Histograms:\n{}", sb);
+ log.info(BenchmarkConstants.SINGLE_LINE);
+ }
+
+ /**
+ * Generate JSON histograms
+ *
+ * @param results Benchmark results
+ * @return JSON string
+ */
+ public String writeJSONHistograms(Results results) {
+ Map map = new HashMap<>();
+ map.put("completed", results.getSuccess());
+ map.put("aborted", results.getAbort());
+ map.put("rejected", results.getRetry());
+ map.put("unexpected", results.getError());
+ return JSONUtil.toJSONString(map);
+ }
+
+ private void writeRawResults(ResultWriter rw, ResultsConfiguration config, String baseFileName)
+ throws Exception {
+ String rawFileName = baseFileName + ".raw.csv";
+ try (PrintStream ps =
+ new PrintStream(FileUtil.joinPath(config.getOutputDirectory(), rawFileName))) {
+ log.info("Output Raw data into file: {}", rawFileName);
+ rw.writeRaw(config.getActiveTxTypes(), ps);
+ }
+ }
+
+ private void writeSamples(ResultWriter rw, ResultsConfiguration config, String baseFileName)
+ throws Exception {
+ String sampleFileName = baseFileName + ".samples.csv";
+ try (PrintStream ps =
+ new PrintStream(FileUtil.joinPath(config.getOutputDirectory(), sampleFileName))) {
+ log.info("Output samples into file: {}", sampleFileName);
+ rw.writeSamples(ps);
+ }
+ }
+
+ private void writeSummary(ResultWriter rw, ResultsConfiguration config, String baseFileName)
+ throws Exception {
+ String summaryFileName = baseFileName + ".summary.json";
+ try (PrintStream ps =
+ new PrintStream(FileUtil.joinPath(config.getOutputDirectory(), summaryFileName))) {
+ log.info("Output summary data into file: {}", summaryFileName);
+ rw.writeSummary(ps);
+ }
+ }
+
+ private void writeParams(ResultWriter rw, ResultsConfiguration config, String baseFileName)
+ throws Exception {
+ String paramsFileName = baseFileName + ".params.json";
+ try (PrintStream ps =
+ new PrintStream(FileUtil.joinPath(config.getOutputDirectory(), paramsFileName))) {
+ log.info("Output DBMS parameters into file: {}", paramsFileName);
+ rw.writeParams(ps);
+ }
+ }
+
+ private void writeMetrics(ResultWriter rw, ResultsConfiguration config, String baseFileName)
+ throws Exception {
+ String metricsFileName = baseFileName + ".metrics.json";
+ try (PrintStream ps =
+ new PrintStream(FileUtil.joinPath(config.getOutputDirectory(), metricsFileName))) {
+ log.info("Output DBMS metrics into file: {}", metricsFileName);
+ rw.writeMetrics(ps);
+ }
+ }
+
+ private void writeConfig(ResultWriter rw, ResultsConfiguration config, String baseFileName)
+ throws Exception {
+ String configFileName = baseFileName + ".config.xml";
+ try (PrintStream ps =
+ new PrintStream(FileUtil.joinPath(config.getOutputDirectory(), configFileName))) {
+ log.info("Output benchmark config into file: {}", configFileName);
+ rw.writeConfig(ps);
+ }
+ }
+
+ private void writeResultsCsv(ResultWriter rw, ResultsConfiguration config, String baseFileName)
+ throws Exception {
+ String resultsFileName = baseFileName + ".results.csv";
+ try (PrintStream ps =
+ new PrintStream(FileUtil.joinPath(config.getOutputDirectory(), resultsFileName))) {
+ log.info(
+ "Output results into file: {} with window size {}",
+ resultsFileName,
+ config.getWindowSize());
+ rw.writeResults(config.getWindowSize(), ps);
+ }
+ }
+
+ private void writePerTransactionResults(
+ ResultWriter rw, ResultsConfiguration config, String baseFileName) throws Exception {
+ for (TransactionType t : config.getActiveTxTypes()) {
+ String fileName = baseFileName + ".results." + t.getName() + ".csv";
+ try (PrintStream ps =
+ new PrintStream(FileUtil.joinPath(config.getOutputDirectory(), fileName))) {
+ rw.writeResults(config.getWindowSize(), ps, t);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/execution/TransactionTypeLoader.java b/src/main/java/com/oltpbenchmark/execution/TransactionTypeLoader.java
new file mode 100644
index 0000000..75ab723
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/execution/TransactionTypeLoader.java
@@ -0,0 +1,193 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.execution;
+
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.api.TransactionType;
+import com.oltpbenchmark.api.TransactionTypes;
+import java.util.ArrayList;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.configuration2.XMLConfiguration;
+
+/** Handles loading transaction types from configuration. */
+@Slf4j
+public class TransactionTypeLoader {
+
+ /**
+ * Load transaction types for a benchmark
+ *
+ * @param xmlConfig XML configuration
+ * @param pluginXpathString Plugin xpath string for XML queries
+ * @param bench Benchmark module
+ * @param lastTxnId Last transaction ID from previous benchmarks
+ * @return Transaction types and list of active transaction types
+ */
+ public LoaderResult loadTransactionTypes(
+ XMLConfiguration xmlConfig, String pluginXpathString, BenchmarkModule bench, int lastTxnId) {
+
+ List ttypes = new ArrayList<>();
+ List activeTXTypes = new ArrayList<>();
+ ttypes.add(TransactionType.INVALID);
+
+ int numTxnTypes = getNumTransactionTypes(xmlConfig, pluginXpathString);
+ int txnIdOffset = lastTxnId;
+
+ for (int i = 1; i <= numTxnTypes; i++) {
+ String key = "transactiontypes" + pluginXpathString + "/transactiontype[" + i + "]";
+ TransactionType txnType = loadTransactionType(xmlConfig, key, bench, i, txnIdOffset);
+
+ // Keep a reference for filtering
+ activeTXTypes.add(txnType);
+
+ // Add a ref for the active TTypes in this benchmark
+ ttypes.add(txnType);
+ }
+
+ // Wrap the list of transactions
+ TransactionTypes tt = new TransactionTypes(ttypes);
+ log.debug("Using the following transaction types: {}", tt);
+
+ return new LoaderResult(tt, activeTXTypes, lastTxnId + numTxnTypes);
+ }
+
+ /**
+ * Load transaction groupings
+ *
+ * @param xmlConfig XML configuration
+ * @param pluginXpathString Plugin xpath string
+ * @param numTxnTypes Number of transaction types
+ */
+ public void loadTransactionGroupings(
+ XMLConfiguration xmlConfig, String pluginXpathString, int numTxnTypes) {
+
+ int numGroupings =
+ xmlConfig
+ .configurationsAt("transactiontypes" + pluginXpathString + "/groupings/grouping")
+ .size();
+
+ log.debug("Num groupings: {}", numGroupings);
+
+ for (int i = 1; i < numGroupings + 1; i++) {
+ String key = "transactiontypes" + pluginXpathString + "/groupings/grouping[" + i + "]";
+ validateGrouping(xmlConfig, key, numTxnTypes);
+ }
+ }
+
+ private int getNumTransactionTypes(XMLConfiguration xmlConfig, String pluginXpathString) {
+ int numTxnTypes =
+ xmlConfig
+ .configurationsAt("transactiontypes" + pluginXpathString + "/transactiontype")
+ .size();
+
+ // if it is a single workload run, w/o attribute is used
+ if (numTxnTypes == 0) {
+ String fallbackTest = "[not(@bench)]";
+ numTxnTypes =
+ xmlConfig.configurationsAt("transactiontypes" + fallbackTest + "/transactiontype").size();
+ }
+
+ return numTxnTypes;
+ }
+
+ private TransactionType loadTransactionType(
+ XMLConfiguration xmlConfig, String key, BenchmarkModule bench, int index, int txnIdOffset) {
+
+ String txnName = xmlConfig.getString(key + "/name");
+
+ // Get ID if specified; else use index
+ int txnId = index;
+ if (xmlConfig.containsKey(key + "/id")) {
+ txnId = xmlConfig.getInt(key + "/id");
+ }
+
+ long preExecutionWait = 0;
+ if (xmlConfig.containsKey(key + "/preExecutionWait")) {
+ preExecutionWait = xmlConfig.getLong(key + "/preExecutionWait");
+ }
+
+ long postExecutionWait = 0;
+ if (xmlConfig.containsKey(key + "/postExecutionWait")) {
+ postExecutionWait = xmlConfig.getLong(key + "/postExecutionWait");
+ }
+
+ return bench.initTransactionType(
+ txnName, txnId + txnIdOffset, preExecutionWait, postExecutionWait);
+ }
+
+ private void validateGrouping(XMLConfiguration xmlConfig, String key, int numTxnTypes) {
+ // Get the name for the grouping and make sure it's valid
+ String groupingName = xmlConfig.getString(key + "/name").toLowerCase();
+
+ if (!groupingName.matches("^[a-z]\\w*$")) {
+ log.error(
+ String.format(
+ "Grouping name \"%s\" is invalid. Must begin with a letter and contain only"
+ + " alphanumeric characters.",
+ groupingName));
+ System.exit(-1);
+ } else if (groupingName.equals("all")) {
+ log.error("Grouping name \"all\" is reserved. Please pick a different name.");
+ System.exit(-1);
+ }
+
+ // Get the weights for this grouping and make sure that there
+ // is an appropriate number of them
+ String[] groupingWeights = xmlConfig.getString(key + "/weights").split("\\s*,\\s*");
+
+ if (groupingWeights.length != numTxnTypes) {
+ log.error(
+ String.format(
+ "Grouping \"%s\" has %d weights, but there are %d transactions in this"
+ + " benchmark.",
+ groupingName, groupingWeights.length, numTxnTypes));
+ System.exit(-1);
+ }
+
+ log.debug(
+ "Creating grouping with name, weights: {}, {}",
+ groupingName,
+ java.util.Arrays.toString(groupingWeights));
+ }
+
+ /** Result of loading transaction types */
+ public static class LoaderResult {
+ private final TransactionTypes transactionTypes;
+ private final List activeTxTypes;
+ private final int lastTxnId;
+
+ public LoaderResult(
+ TransactionTypes transactionTypes, List activeTxTypes, int lastTxnId) {
+ this.transactionTypes = transactionTypes;
+ this.activeTxTypes = activeTxTypes;
+ this.lastTxnId = lastTxnId;
+ }
+
+ public TransactionTypes getTransactionTypes() {
+ return transactionTypes;
+ }
+
+ public List getActiveTxTypes() {
+ return activeTxTypes;
+ }
+
+ public int getLastTxnId() {
+ return lastTxnId;
+ }
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/execution/WorkloadExecutor.java b/src/main/java/com/oltpbenchmark/execution/WorkloadExecutor.java
new file mode 100644
index 0000000..fa32798
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/execution/WorkloadExecutor.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2020 by OLTPBenchmark Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.oltpbenchmark.execution;
+
+import com.oltpbenchmark.Results;
+import com.oltpbenchmark.ThreadBench;
+import com.oltpbenchmark.WorkloadConfiguration;
+import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.api.Worker;
+import com.oltpbenchmark.util.MonitorInfo;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+
+/** Handles the execution of benchmark workloads. */
+@Slf4j
+public class WorkloadExecutor {
+
+ /**
+ * Execute the workload for all benchmarks
+ *
+ * @param benchmarks List of benchmark modules to execute
+ * @param monitorInfo Monitoring configuration
+ * @return Results of the benchmark execution
+ * @throws IOException if I/O operation fails
+ */
+ public Results execute(List benchmarks, MonitorInfo monitorInfo)
+ throws IOException {
+
+ List> workers = createWorkers(benchmarks);
+ List workConfs = new ArrayList<>();
+
+ for (BenchmarkModule bench : benchmarks) {
+ workConfs.add(bench.getWorkloadConfiguration());
+ logBenchmarkStart(bench);
+ }
+
+ Results r = ThreadBench.runRateLimitedBenchmark(workers, workConfs, monitorInfo);
+
+ log.info(BenchmarkConstants.SINGLE_LINE);
+ log.info("Rate limited reqs/s: {}", r);
+
+ return r;
+ }
+
+ /**
+ * Create workers for all benchmarks
+ *
+ * @param benchmarks List of benchmark modules
+ * @return List of workers for all benchmarks
+ * @throws IOException if I/O operation fails
+ */
+ private List> createWorkers(List benchmarks) throws IOException {
+ List> workers = new ArrayList<>();
+
+ for (BenchmarkModule bench : benchmarks) {
+ int terminals = bench.getWorkloadConfiguration().getTerminals();
+ log.info("Creating {} virtual terminals...", terminals);
+ workers.addAll(bench.makeWorkers());
+ }
+
+ return workers;
+ }
+
+ /**
+ * Log benchmark start information
+ *
+ * @param bench Benchmark module
+ */
+ private void logBenchmarkStart(BenchmarkModule bench) {
+ int numPhases = bench.getWorkloadConfiguration().getNumberOfPhases();
+ log.info(
+ String.format(
+ "Launching the %s Benchmark with %s Phase%s...",
+ bench.getBenchmarkName().toUpperCase(), numPhases, (numPhases > 1 ? "s" : "")));
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/util/ConnectionRateLimiterUtil.java b/src/main/java/com/oltpbenchmark/util/ConnectionRateLimiterUtil.java
new file mode 100644
index 0000000..cb890f9
--- /dev/null
+++ b/src/main/java/com/oltpbenchmark/util/ConnectionRateLimiterUtil.java
@@ -0,0 +1,23 @@
+package com.oltpbenchmark.util;
+
+import com.google.common.util.concurrent.RateLimiter;
+
+/** Utility for rate limiting connection creation. */
+public final class ConnectionRateLimiterUtil {
+ private static final double DEFAULT_CONNECTION_RATE_LIMIT = 10.0;
+ private static RateLimiter rateLimiter;
+
+ /**
+ * Get a rate limiter for connection creation.
+ *
+ * @param rate The rate limit in connections per second
+ * @return The rate limiter
+ */
+ public static synchronized RateLimiter getRateLimiter(double rate) {
+ if (rateLimiter == null) {
+ double effectiveRate = rate > 0 ? rate : DEFAULT_CONNECTION_RATE_LIMIT;
+ rateLimiter = RateLimiter.create(effectiveRate);
+ }
+ return rateLimiter;
+ }
+}
diff --git a/src/main/java/com/oltpbenchmark/util/ConnectionUtil.java b/src/main/java/com/oltpbenchmark/util/ConnectionUtil.java
index 85a0d0f..8795d73 100644
--- a/src/main/java/com/oltpbenchmark/util/ConnectionUtil.java
+++ b/src/main/java/com/oltpbenchmark/util/ConnectionUtil.java
@@ -1,24 +1,43 @@
package com.oltpbenchmark.util;
import com.google.common.util.concurrent.RateLimiter;
+import com.oltpbenchmark.WorkloadConfiguration;
import com.oltpbenchmark.api.BenchmarkModule;
+import com.oltpbenchmark.types.DatabaseType;
import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
-public class ConnectionUtil {
+/** Connection utils with retries. */
+public final class ConnectionUtil {
private static final double DEFAULT_CONNECTION_RATE_LIMIT = 10.0;
private static final Logger LOG = LoggerFactory.getLogger(ConnectionUtil.class);
private static RateLimiter rateLimiter = null;
+ /** Connection operation string. */
+ private static final String CONNECTION_OPERATION_SUCCESS = "CONNECTION-SUCCESS";
+
+ /** Connection failed operation string. */
+ private static final String CONNECTION_OPERATION_FAILED = "CONNECTION-FAILED";
+
public static Connection makeConnectionWithRetry(BenchmarkModule benchmark) {
+ final WorkloadConfiguration wrkld = benchmark.getWorkloadConfiguration();
+ final DatabaseType dbType = wrkld.getDatabaseType();
int attempts = 0;
while (attempts <= benchmark.getWorkloadConfiguration().getMaxRetries()) {
+ getRateLimiter().acquire();
+ long startTimeNanos = System.nanoTime();
try {
- getRateLimiter().acquire();
Connection newConnection = benchmark.makeConnection();
+ if (DatabaseType.AURORADSQL.equals(dbType)) {
+ populateSessionIdOnThreadContext(newConnection);
+ }
return newConnection;
} catch (Exception e) {
attempts++;
@@ -48,7 +67,7 @@ private static long calExpDelay(int attempts) {
long delay = (long) (baseDelay * Math.pow(2, attempts));
delay = (long) (delay * (1 + jitterFactor * Math.random()));
- delay = Math.min(delay, 4000);
+ delay = Math.min(delay, 4_000); // Cap the delay to 4s
return delay;
}
@@ -59,4 +78,17 @@ private static synchronized RateLimiter getRateLimiter() {
}
return rateLimiter;
}
+
+ private static void populateSessionIdOnThreadContext(Connection connection) throws SQLException {
+ try (Statement statement = connection.createStatement()) {
+ try (ResultSet rs = statement.executeQuery("SELECT sys.current_session_id()")) {
+ if (rs.next()) {
+ String sessionId = rs.getString(1);
+ MDC.put("DBSessionId", sessionId);
+ LOG.info(
+ "Populated session id {} for thread {}", sessionId, Thread.currentThread().getName());
+ }
+ }
+ }
+ }
}
diff --git a/src/main/java/com/oltpbenchmark/util/IAMUtil.java b/src/main/java/com/oltpbenchmark/util/IAMUtil.java
index a47a8d1..15b97e3 100644
--- a/src/main/java/com/oltpbenchmark/util/IAMUtil.java
+++ b/src/main/java/com/oltpbenchmark/util/IAMUtil.java
@@ -4,7 +4,7 @@
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.core.exception.SdkClientException;
-import software.amazon.awssdk.regions.providers.AwsRegionProviderChain;
+import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
import software.amazon.awssdk.services.dsql.DsqlUtilities;
@@ -15,40 +15,38 @@ public class IAMUtil {
private static final String ADMIN_USERNAME = "admin";
public static String generateAuroraDsqlPasswordToken(String url, String username) {
+ return generateAuroraDsqlPasswordToken(url, username, null);
+ }
+
+ public static String generateAuroraDsqlPasswordToken(String url, String username, String region) {
+ if (region == null) {
+ Region defaultRegion = DefaultAwsRegionProviderChain.builder().build().getRegion();
+ return generateAuroraDsqlPasswordToken(
+ url,
+ username,
+ DefaultCredentialsProvider.builder().reuseLastProviderEnabled(false).build(),
+ defaultRegion);
+ }
return generateAuroraDsqlPasswordToken(
url,
username,
DefaultCredentialsProvider.builder().reuseLastProviderEnabled(false).build(),
- DefaultAwsRegionProviderChain.builder().build());
+ Region.of(region));
}
public static String generateAuroraDsqlPasswordToken(
- String url,
- String username,
- AwsCredentialsProvider credentialsProvider,
- AwsRegionProviderChain regionProvider) {
+ String url, String username, AwsCredentialsProvider credentialsProvider, Region region) {
DsqlUtilities utilities =
- DsqlUtilities.builder()
- .region(regionProvider.getRegion())
- .credentialsProvider(credentialsProvider)
- .build();
+ DsqlUtilities.builder().region(region).credentialsProvider(credentialsProvider).build();
try {
IAMUtil.validateUrl(url);
String host = url.split("//")[1].split(":")[0];
return username.equals(ADMIN_USERNAME)
? utilities.generateDbConnectAdminAuthToken(
- builder ->
- builder
- .hostname(host)
- .region(regionProvider.getRegion())
- .expiresIn(DEFAULT_VALIDITY))
+ builder -> builder.hostname(host).region(region).expiresIn(DEFAULT_VALIDITY))
: utilities.generateDbConnectAuthToken(
- builder ->
- builder
- .hostname(host)
- .region(regionProvider.getRegion())
- .expiresIn(DEFAULT_VALIDITY));
+ builder -> builder.hostname(host).region(region).expiresIn(DEFAULT_VALIDITY));
} catch (SdkClientException e) {
throw new RuntimeException(e);
}
diff --git a/src/main/java/com/oltpbenchmark/util/TimeUtil.java b/src/main/java/com/oltpbenchmark/util/TimeUtil.java
index 82675c8..3c7e887 100644
--- a/src/main/java/com/oltpbenchmark/util/TimeUtil.java
+++ b/src/main/java/com/oltpbenchmark/util/TimeUtil.java
@@ -47,4 +47,27 @@ public static String getCurrentTimeString() {
public static Timestamp getCurrentTime() {
return new Timestamp(System.currentTimeMillis());
}
+
+ /**
+ * Calculate exponential backoff delay with jitter!
+ *
+ * NOTE: Added to time util class to avoid additional class just for one method.
+ *
+ * @param attempts
+ * @return Exponential Delay
+ */
+ public static long calExpDelay(int attempts) {
+ long baseDelay = 1000; // Initial delay in milliseconds
+ double jitterFactor = 1.0; // Jitter factor (between 0 and 1)
+
+ long delay = (long) (baseDelay * Math.pow(2, attempts));
+ delay = (long) (delay * (1 + jitterFactor * Math.random()));
+ delay = Math.min(delay, 4000);
+
+ return delay;
+ }
+
+ public static long getTimeDiffInMicro(long startNano, long endNano) {
+ return (endNano - startNano) / 1000;
+ }
}
diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties
index 3b876e2..4a3ed73 100644
--- a/src/main/resources/log4j.properties
+++ b/src/main/resources/log4j.properties
@@ -5,7 +5,7 @@ log4j.rootLogger.layout=org.apache.log4j.PatternLayout
# A1 is set to be a ConsoleAppender.
log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
-log4j.appender.A1.layout.ConversionPattern=[%-5p] %d [%t] %x %c %M - %m%n
+log4j.appender.A1.layout.ConversionPattern=[%-5p] %d [%t] [DBSessionId=%X{DBSessionId}] %x %c %M - %m%n
log4j.appender.A1.ImmediateFlush=true
log4j.logger.com.oltpbenchmark=INFO
diff --git a/src/test/java/com/oltpbenchmark/benchmarks/templated/TestTemplatedWorker.java b/src/test/java/com/oltpbenchmark/benchmarks/templated/TestTemplatedWorker.java
index 026f8c0..43bb496 100644
--- a/src/test/java/com/oltpbenchmark/benchmarks/templated/TestTemplatedWorker.java
+++ b/src/test/java/com/oltpbenchmark/benchmarks/templated/TestTemplatedWorker.java
@@ -3,11 +3,11 @@
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
-import com.oltpbenchmark.DBWorkload;
import com.oltpbenchmark.WorkloadConfiguration;
import com.oltpbenchmark.api.AbstractTestWorker;
import com.oltpbenchmark.api.Procedure;
import com.oltpbenchmark.benchmarks.tpcc.TPCCBenchmark;
+import com.oltpbenchmark.execution.ConfigurationLoader;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.util.ArrayList;
@@ -43,7 +43,7 @@ public TestTemplatedWorker() {
public static void setWorkloadConfigXml(WorkloadConfiguration workConf) {
// Load the configuration file so we can parse the query_template_file value.
try {
- XMLConfiguration xmlConf = DBWorkload.buildConfiguration(SAMPLE_TEMPLATED_CONFIG);
+ XMLConfiguration xmlConf = ConfigurationLoader.loadXMLConfiguration(SAMPLE_TEMPLATED_CONFIG);
workConf.setXmlConfig(xmlConf);
} catch (ConfigurationException ex) {
LOG.error("Error loading configuration: " + SAMPLE_TEMPLATED_CONFIG, ex);
diff --git a/src/test/java/com/oltpbenchmark/util/TestIAMUtil.java b/src/test/java/com/oltpbenchmark/util/TestIAMUtil.java
index b9f58a1..a30a04a 100644
--- a/src/test/java/com/oltpbenchmark/util/TestIAMUtil.java
+++ b/src/test/java/com/oltpbenchmark/util/TestIAMUtil.java
@@ -46,7 +46,7 @@ public void testGenerateAuroraDsqlPasswordTokenInvalidUrl() {
RuntimeException.class,
() ->
IAMUtil.generateAuroraDsqlPasswordToken(
- "htp:/bad-url", VALID_ADMIN_USERNAME, credentialsProvider, regionProvider));
+ "htp:/bad-url", VALID_ADMIN_USERNAME, credentialsProvider, Region.US_EAST_2));
}
@Test
@@ -56,6 +56,6 @@ public void testGenerateAuroraDsqlPasswordTokenAWSCredentialProviderSdkClientExc
RuntimeException.class,
() ->
IAMUtil.generateAuroraDsqlPasswordToken(
- "htp:/bad-url", VALID_ADMIN_USERNAME, credentialsProvider, regionProvider));
+ "htp:/bad-url", VALID_ADMIN_USERNAME, credentialsProvider, Region.US_EAST_2));
}
}