Skip to content

Commit 2dda027

Browse files
ErikZigoErikZigo
authored
Merge pull request #397 from keboola/erik-table-create-KBC-202
Synapse table create + drop
2 parents 904ad14 + c03f489 commit 2dda027

7 files changed

Lines changed: 313 additions & 5 deletions

File tree

‎phpunit.xml.dist‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,4 +84,10 @@
8484
<directory>tests/Backend/Mixed</directory>
8585
<exclude>tests/Backend/Mixed/Workspaces/S3WorkspacesTest.php</exclude>
8686
</testsuite>
87+
88+
<testsuite name="backend-synapse-part-1">
89+
<file>tests/Backend/CommonPart1/BucketsTest.php</file>
90+
<file>tests/Backend/Synapse/CreateTableTest.php</file>
91+
<file>tests/Backend/Synapse/ImportExportCommonTest.php</file>
92+
</testsuite>
8793
</phpunit>

‎src/Keboola/StorageApi/Client.php‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1578,7 +1578,11 @@ private function uploadSlicedFileToAbs(
15781578
foreach ($slices as $slice) {
15791579
$blobClient->createBlockBlob(
15801580
$prepareResult['absUploadParams']['container'],
1581-
basename($slice),
1581+
sprintf(
1582+
'%s%s',
1583+
$prepareResult['absUploadParams']['blobName'],
1584+
basename($slice)
1585+
),
15821586
fopen($slice, 'r')
15831587
);
15841588
}
@@ -1589,7 +1593,13 @@ private function uploadSlicedFileToAbs(
15891593

15901594
foreach ($slices as $filePath) {
15911595
$manifest['entries'][] = [
1592-
"url" => "azure://" . $prepareResult['absUploadParams']['accountName'] . 'blob.core.windows.net/' . $prepareResult['absUploadParams']['container'] . '/' . basename($filePath),
1596+
'url' => sprintf(
1597+
'azure://%sblob.core.windows.net/%s/%s%s',
1598+
$prepareResult['absUploadParams']['accountName'],
1599+
$prepareResult['absUploadParams']['container'],
1600+
$prepareResult['absUploadParams']['blobName'],
1601+
basename($filePath)
1602+
),
15931603
];
15941604
}
15951605
$blobClient->createBlockBlob($prepareResult['absUploadParams']['container'], $prepareResult['absUploadParams']['blobName'] . 'manifest', json_encode($manifest));

‎tests/Backend/CommonPart1/CreateTableTest.php‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -372,11 +372,14 @@ public function testRowNumberAmbiguity()
372372
$this->assertNotEmpty($tableId);
373373
}
374374

375-
public function testCreateTableFromSlicedFile()
375+
/**
376+
* @dataProvider createTableFromSlicedFileData
377+
*/
378+
public function testCreateTableFromSlicedFile($fileName)
376379
{
377380
$uploadOptions = new \Keboola\StorageApi\Options\FileUploadOptions();
378381
$uploadOptions
379-
->setFileName('languages_')
382+
->setFileName($fileName)
380383
->setIsSliced(true)
381384
->setIsEncrypted(false);
382385
$slices = [
@@ -406,4 +409,19 @@ public function invalidPrimaryKeys()
406409
array('idus'),
407410
);
408411
}
412+
413+
public function createTableFromSlicedFileData()
414+
{
415+
return [
416+
'same prefix as slice' => [
417+
'languages',
418+
],
419+
'same prefix (due webalize) as slice' => [
420+
'languages_',
421+
],
422+
'other' => [
423+
'other',
424+
],
425+
];
426+
}
409427
}

‎tests/Backend/CommonPart1/ImportExportCommonTest.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ public function incrementalImportPkDedupeData()
155155
public function testTableImportColumnsCaseInsensitive()
156156
{
157157
$tokenData = $this->_client->verifyToken();
158-
if ($tokenData['owner']['defaultBackend'] == self::BACKEND_SNOWFLAKE) {
158+
if (in_array($tokenData['owner']['defaultBackend'], [self::BACKEND_SNOWFLAKE, self::BACKEND_SYNAPSE])) {
159159
return;
160160
}
161161

‎tests/Backend/Synapse/CreateTableTest.php‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
<?php
2+
3+
4+
5+
namespace Keboola\Test\Backend\Synapse;
6+
7+
use Keboola\Csv\CsvFile;
8+
use Keboola\StorageApi\Client;
9+
use Keboola\StorageApi\ClientException;
10+
use Keboola\StorageApi\Exception;
11+
use Keboola\StorageApi\Workspaces;
12+
use Keboola\Test\Backend\Workspaces\Backend\WorkspaceBackendFactory;
13+
use Keboola\Test\Backend\Workspaces\WorkspacesTestCase;
14+
use Keboola\Test\Backend\CommonPart1\CreateTableTest as CommonCreateTableTest;
15+
16+
class CreateTableTest extends CommonCreateTableTest
17+
{
18+
/**
19+
* @dataProvider tableCreateData
20+
* @param $createFile
21+
*/
22+
public function testTableCreate($tableName, $createFile, $expectationFile, $async, $options = array())
23+
{
24+
$createMethod = $async ? 'createTableAsync' : 'createTable';
25+
$tableId = $this->_client->{$createMethod}(
26+
$this->getTestBucketId(self::STAGE_IN),
27+
$tableName,
28+
new CsvFile($createFile),
29+
$options
30+
);
31+
$table = $this->_client->getTable($tableId);
32+
33+
$this->assertArrayHasKey('displayName', $table['bucket']);
34+
35+
$expectationFileCsv = new CsvFile($expectationFile);
36+
37+
$this->assertEquals($tableId, $table['id']);
38+
$this->assertEquals($tableName, $table['name']);
39+
$this->assertEquals($tableName, $table['displayName'], 'display name is same as name');
40+
$this->assertNotEmpty($table['created']);
41+
$this->assertNotEmpty($table['lastChangeDate']);
42+
$this->assertNotEmpty($table['lastImportDate']);
43+
$this->assertEquals($expectationFileCsv->getHeader(), $table['columns']);
44+
$this->assertEmpty($table['indexedColumns']);
45+
$this->assertNotEquals('0000-00-00 00:00:00', $table['created']);
46+
$this->assertNotEmpty($table['dataSizeBytes']);
47+
48+
// // @TODO not implemented yet
49+
// $this->assertLinesEqualsSorted(
50+
// file_get_contents($expectationFile),
51+
// $this->_client->getTableDataPreview($tableId),
52+
// 'initial data imported into table'
53+
// );
54+
55+
$displayName = 'Romanov-display-name';
56+
$tableId = $this->_client->updateTable(
57+
$tableId,
58+
[
59+
'displayName' => $displayName,
60+
]
61+
);
62+
63+
$table = $this->_client->getTable($tableId);
64+
65+
$this->assertEquals($displayName, $table['displayName']);
66+
67+
// rename table to same name it already has should succeed
68+
$this->_client->updateTable(
69+
$tableId,
70+
[
71+
'displayName' => $displayName,
72+
]
73+
);
74+
75+
try {
76+
$this->_client->updateTable(
77+
$tableId,
78+
[
79+
'displayName' => '_wrong-display-name',
80+
]
81+
);
82+
} catch (\Keboola\StorageApi\ClientException $e) {
83+
$this->assertEquals(
84+
'Invalid data - displayName: Cannot start with underscore.',
85+
$e->getMessage()
86+
);
87+
$this->assertEquals('storage.tables.validation', $e->getStringCode());
88+
$this->assertEquals(400, $e->getCode());
89+
}
90+
91+
$tableNameAnother = $tableName . '_another';
92+
$anotherTableId = $this->_client->{$createMethod}(
93+
$this->getTestBucketId(self::STAGE_IN),
94+
$tableNameAnother,
95+
new CsvFile($createFile),
96+
$options
97+
);
98+
try {
99+
$this->_client->updateTable(
100+
$anotherTableId,
101+
[
102+
'displayName' => $displayName,
103+
]
104+
);
105+
$this->fail('Renaming another table to existing displayname should fail');
106+
} catch (\Keboola\StorageApi\ClientException $e) {
107+
$this->assertEquals(
108+
sprintf(
109+
'The table "%s" in the bucket already has the same display name "%s".',
110+
$table['name'],
111+
$displayName
112+
),
113+
$e->getMessage()
114+
);
115+
$this->assertEquals('storage.buckets.tableAlreadyExists', $e->getStringCode());
116+
$this->assertEquals(400, $e->getCode());
117+
}
118+
}
119+
120+
public function testRowNumberAmbiguity()
121+
{
122+
$importFile = __DIR__ . '/../../_data/column-name-row-number.csv';
123+
124+
// create and import data into source table
125+
$tableId = $this->_client->createTable(
126+
$this->getTestBucketId(),
127+
'column-name-row-number',
128+
new CsvFile($importFile)
129+
);
130+
131+
// // @TODO not implemented yet
132+
// // this used to fail because of the column named row_number
133+
// $this->_client->createTablePrimaryKey($tableId, ['id']);
134+
// $this->assertNotEmpty($tableId);
135+
}
136+
}

‎tests/Backend/Synapse/ImportExportCommonTest.php‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
<?php
2+
3+
4+
5+
namespace Keboola\Test\Backend\Synapse;
6+
7+
use Keboola\Csv\CsvFile;
8+
use Keboola\StorageApi\Client;
9+
use Keboola\StorageApi\ClientException;
10+
use Keboola\StorageApi\Exception;
11+
use Keboola\StorageApi\Workspaces;
12+
use Keboola\Test\Backend\Workspaces\Backend\WorkspaceBackendFactory;
13+
use Keboola\Test\Backend\Workspaces\WorkspacesTestCase;
14+
use Keboola\Test\Backend\CommonPart1\ImportExportCommonTest as CommonImportExportTest;
15+
16+
class ImportExportCommonTest extends CommonImportExportTest
17+
{
18+
/**
19+
* @dataProvider tableImportData
20+
* @param $importFileName
21+
*/
22+
public function testTableImportExport(CsvFile $importFile, $expectationsFileName, $colNames, $format = 'rfc', $createTableOptions = array())
23+
{
24+
$expectationsFile = __DIR__ . '/../../_data/' . $expectationsFileName;
25+
$tableId = $this->_client->createTable($this->getTestBucketId(self::STAGE_IN), 'languages-2', $importFile, $createTableOptions);
26+
27+
$result = $this->_client->writeTable($tableId, $importFile);
28+
$table = $this->_client->getTable($tableId);
29+
30+
$this->assertEmpty($result['warnings']);
31+
$this->assertEquals($colNames, array_values($result['importedColumns']), 'columns');
32+
$this->assertEmpty($result['transaction']);
33+
$this->assertNotEmpty($table['dataSizeBytes']);
34+
$this->assertNotEmpty($result['totalDataSizeBytes']);
35+
36+
// @TODO not implemented yet
37+
// // compare data
38+
// $this->assertLinesEqualsSorted(file_get_contents($expectationsFile), $this->_client->getTableDataPreview($tableId, array(
39+
// 'format' => $format,
40+
// )), 'imported data comparsion');
41+
//
42+
// // incremental
43+
// $result = $this->_client->writeTable($tableId, $importFile, array(
44+
// 'incremental' => true,
45+
// ));
46+
// $this->assertNotEmpty($result['totalDataSizeBytes']);
47+
}
48+
49+
/**
50+
* @dataProvider tableImportData
51+
* @param $importFileName
52+
*/
53+
public function testTableAsyncImportExport(CsvFile $importFile, $expectationsFileName, $colNames, $format = 'rfc', $createTableOptions = array())
54+
{
55+
$expectationsFile = __DIR__ . '/../../_data/' . $expectationsFileName;
56+
$tableId = $this->_client->createTable($this->getTestBucketId(self::STAGE_IN), 'languages-3', $importFile, $createTableOptions);
57+
58+
$result = $this->_client->writeTableAsync($tableId, $importFile);
59+
$table = $this->_client->getTable($tableId);
60+
61+
$this->assertEmpty($result['warnings']);
62+
$this->assertEquals($colNames, array_values($result['importedColumns']), 'columns');
63+
$this->assertEmpty($result['transaction']);
64+
$this->assertNotEmpty($table['dataSizeBytes']);
65+
$this->assertNotEmpty($result['totalDataSizeBytes']);
66+
67+
// @TODO not implemented yet
68+
// // compare data
69+
// $this->assertLinesEqualsSorted(file_get_contents($expectationsFile), $this->_client->getTableDataPreview($tableId, array(
70+
// 'format' => $format,
71+
// )), 'imported data comparsion');
72+
//
73+
// // incremental
74+
//
75+
// $result = $this->_client->writeTableAsync($tableId, $importFile, array(
76+
// 'incremental' => true,
77+
// ));
78+
// $this->assertNotEmpty($result['totalDataSizeBytes']);
79+
}
80+
81+
/**
82+
* @dataProvider incrementalImportPkDedupeData
83+
* @param $createFile
84+
* @param $primaryKey
85+
* @param $expectationFileAfterCreate
86+
* @param $incrementFile
87+
* @param $expectationFinal
88+
*/
89+
public function testIncrementalImportPkDedupe($createFile, $primaryKey, $expectationFileAfterCreate, $incrementFile, $expectationFinal)
90+
{
91+
92+
$tableId = $this->_client->createTableAsync($this->getTestBucketId(self::STAGE_IN), 'pk', $createFile, [
93+
'primaryKey' => $primaryKey,
94+
]);
95+
96+
// @TODO not implemented yet
97+
// $this->assertLinesEqualsSorted(file_get_contents($expectationFileAfterCreate), $this->_client->getTableDataPreview($tableId));
98+
99+
$this->_client->writeTableAsync($tableId, $incrementFile, [
100+
'incremental' => true,
101+
]);
102+
103+
// @TODO not implemented yet
104+
// $this->assertLinesEqualsSorted(file_get_contents($expectationFinal), $this->_client->getTableDataPreview($tableId));
105+
}
106+
107+
public function testTableImportCreateMissingColumns()
108+
{
109+
$this->markTestSkipped('Adding missing columns for Synapse backend is not supported yet');
110+
}
111+
112+
public function testTableImportFromString()
113+
{
114+
$tableId = $this->_client->createTable($this->getTestBucketId(self::STAGE_IN), 'languages', new \Keboola\Csv\CsvFile(__DIR__ . '/../../_data/languages-headers.csv'));
115+
116+
$lines = '"id","name"';
117+
$lines .= "\n" . '"first","second"' . "\n";
118+
$this->_client->apiPost("storage/tables/$tableId/import", array(
119+
'dataString' => $lines,
120+
));
121+
122+
// @TODO not implemented yet
123+
// $this->assertEquals($lines, $this->_client->getTableDataPreview($tableId, array(
124+
// 'format' => 'rfc',
125+
// )));
126+
}
127+
128+
public function testTableInvalidAsyncImport()
129+
{
130+
$this->markTestSkipped('Adding missing table columns for Synapse backend is not supported yet');
131+
}
132+
133+
public function testTableAsyncExportRepeatedly()
134+
{
135+
$this->markTestSkipped('Exporting table table for Synapse backend is not supported yet');
136+
}
137+
}

‎tests/StorageApiTestCase.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ abstract class StorageApiTestCase extends \PHPUnit_Framework_TestCase
2121
{
2222
const BACKEND_REDSHIFT = 'redshift';
2323
const BACKEND_SNOWFLAKE = 'snowflake';
24+
const BACKEND_SYNAPSE = 'synapse';
2425

2526
const STAGE_IN = 'in';
2627
const STAGE_OUT = 'out';

0 commit comments

Comments
 (0)