Skip to content

Commit 12753e3

Browse files
artongeartonge
committed
Migrate metadata as JSON to value as STRING
Signed-off-by: Louis Chemineau <louis@chmn.me>
1 parent 691aa8d commit 12753e3

9 files changed

Lines changed: 221 additions & 24 deletions

File tree

‎apps/dav/lib/Connector/Sabre/FilesPlugin.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -436,7 +436,7 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node)
436436
\OC::$server->get(LoggerInterface::class)->debug('Inefficient fetching of metadata');
437437
}
438438

439-
return json_encode((object)$sizeMetadata->getMetadata(), JSON_THROW_ON_ERROR);
439+
return $sizeMetadata->getValue();
440440
});
441441
}
442442
}

‎core/Migrations/Version24000Date20220404230027.php‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,15 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
5252
'notnull' => true,
5353
'length' => 50,
5454
]);
55-
$table->addColumn('metadata', Types::JSON, [
56-
'notnull' => true,
55+
$table->addColumn('value', Types::STRING, [
56+
'notnull' => false,
57+
'default' => '',
5758
]);
5859
$table->setPrimaryKey(['id', 'group_name'], 'file_metadata_idx');
60+
61+
return $schema;
5962
}
60-
return $schema;
63+
64+
return null;
6165
}
6266
}

‎core/Migrations/Version27000Date20230309104325.php‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* @copyright Copyright (c) 2023 Louis Chmn <louis@chmn.me>
7+
*
8+
* @author Louis Chmn <louis@chmn.me>
9+
*
10+
* @license GNU AGPL version 3 or any later version
11+
*
12+
* This program is free software: you can redistribute it and/or modify
13+
* it under the terms of the GNU Affero General Public License as
14+
* published by the Free Software Foundation, either version 3 of the
15+
* License, or (at your option) any later version.
16+
*
17+
* This program is distributed in the hope that it will be useful,
18+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
19+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20+
* GNU Affero General Public License for more details.
21+
*
22+
* You should have received a copy of the GNU Affero General Public License
23+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
24+
*
25+
*/
26+
27+
namespace OC\Core\Migrations;
28+
29+
use Closure;
30+
use OCP\DB\ISchemaWrapper;
31+
use OCP\DB\QueryBuilder\IQueryBuilder;
32+
use OCP\DB\Types;
33+
use OCP\IDBConnection;
34+
use OCP\Migration\IOutput;
35+
use OCP\Migration\SimpleMigrationStep;
36+
37+
/**
38+
* Migrate oc_file_metadata.metadata as JSON type to oc_file_metadata.value a STRING type
39+
* @see \OC\Metadata\FileMetadata
40+
*/
41+
class Version27000Date20230309104325 extends SimpleMigrationStep {
42+
public function __construct(
43+
private IDBConnection $connection
44+
) {
45+
}
46+
47+
/**
48+
* @param IOutput $output
49+
* @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
50+
* @param array $options
51+
* @return null|ISchemaWrapper
52+
*/
53+
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
54+
/** @var ISchemaWrapper $schema */
55+
$schema = $schemaClosure();
56+
$metadataTable = $schema->getTable('file_metadata');
57+
58+
if ($metadataTable->hasColumn('value')) {
59+
return null;
60+
}
61+
62+
$metadataTable->addColumn('value', Types::STRING, [
63+
'notnull' => false,
64+
'default' => '',
65+
]);
66+
return $schema;
67+
}
68+
69+
70+
/**
71+
* @param IOutput $output
72+
* @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
73+
* @param array $options
74+
* @return void
75+
*/
76+
public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options) {
77+
/** @var ISchemaWrapper $schema */
78+
$schema = $schemaClosure();
79+
$metadataTable = $schema->getTable('file_metadata');
80+
81+
if (!$metadataTable->hasColumn('metadata')) {
82+
return;
83+
}
84+
85+
$updateQuery = $this->connection->getQueryBuilder();
86+
$updateQuery->update('file_metadata')
87+
->set('value', $updateQuery->createParameter('value'))
88+
->where($updateQuery->expr()->eq('id', $updateQuery->createParameter('id')))
89+
->andWhere($updateQuery->expr()->eq('group_name', $updateQuery->createParameter('group_name')));
90+
91+
$selectQuery = $this->connection->getQueryBuilder();
92+
$selectQuery->select('id', 'group_name', 'metadata')
93+
->from('file_metadata')
94+
->orderBy('id', 'ASC')
95+
->setMaxResults(1000);
96+
97+
$offset = 0;
98+
$movedRows = 0;
99+
do {
100+
$movedRows = $this->chunkedCopying($updateQuery, $selectQuery, $offset);
101+
$offset += $movedRows;
102+
} while ($movedRows !== 0);
103+
}
104+
105+
protected function chunkedCopying(IQueryBuilder $updateQuery, IQueryBuilder $selectQuery, int $offset): int {
106+
$this->connection->beginTransaction();
107+
108+
$results = $selectQuery
109+
->setFirstResult($offset)
110+
->executeQuery();
111+
112+
while ($row = $results->fetch()) {
113+
$updateQuery
114+
->setParameter('id', (int)$row['id'])
115+
->setParameter('group_name', $row['group_name'])
116+
->setParameter('value', $row['metadata'])
117+
->executeStatement();
118+
}
119+
120+
$results->closeCursor();
121+
$this->connection->commit();
122+
123+
return $results->rowCount();
124+
}
125+
}

‎core/Migrations/Version27000Date20230309104802.php‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* @copyright Copyright (c) 2023 Louis Chmn <louis@chmn.me>
7+
*
8+
* @author Louis Chmn <louis@chmn.me>
9+
*
10+
* @license GNU AGPL version 3 or any later version
11+
*
12+
* This program is free software: you can redistribute it and/or modify
13+
* it under the terms of the GNU Affero General Public License as
14+
* published by the Free Software Foundation, either version 3 of the
15+
* License, or (at your option) any later version.
16+
*
17+
* This program is distributed in the hope that it will be useful,
18+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
19+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20+
* GNU Affero General Public License for more details.
21+
*
22+
* You should have received a copy of the GNU Affero General Public License
23+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
24+
*
25+
*/
26+
27+
namespace OC\Core\Migrations;
28+
29+
use Closure;
30+
use OCP\DB\ISchemaWrapper;
31+
use OCP\Migration\IOutput;
32+
use OCP\Migration\SimpleMigrationStep;
33+
34+
/**
35+
* Migrate oc_file_metadata.metadata as JSON type to oc_file_metadata.value a STRING type
36+
* @see \OC\Metadata\FileMetadata
37+
*/
38+
class Version27000Date20230309104802 extends SimpleMigrationStep {
39+
/**
40+
* @param IOutput $output
41+
* @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
42+
* @param array $options
43+
* @return null|ISchemaWrapper
44+
*/
45+
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
46+
/** @var ISchemaWrapper $schema */
47+
$schema = $schemaClosure();
48+
$metadataTable = $schema->getTable('file_metadata');
49+
50+
if ($metadataTable->hasColumn('metadata')) {
51+
$metadataTable->dropColumn('metadata');
52+
return $schema;
53+
}
54+
55+
return null;
56+
}
57+
}

‎lib/private/Metadata/FileMetadata.php‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,24 @@
2828
/**
2929
* @method string getGroupName()
3030
* @method void setGroupName(string $groupName)
31-
* @method array getMetadata()
32-
* @method void setMetadata(array $metadata)
31+
* @method string getValue()
32+
* @method void setValue(string $value)
3333
* @see \OC\Core\Migrations\Version240000Date20220404230027
3434
*/
3535
class FileMetadata extends Entity {
3636
protected ?string $groupName = null;
37-
protected ?array $metadata = null;
37+
protected ?string $value = null;
3838

3939
public function __construct() {
4040
$this->addType('groupName', 'string');
41-
$this->addType('metadata', Types::JSON);
41+
$this->addType('value', Types::STRING);
42+
}
43+
44+
public function getDecodedValue(): array {
45+
return json_decode($this->getValue(), true) ?? [];
46+
}
47+
48+
public function setObjectAsValue(array $value): void {
49+
$this->setValue(json_encode($value, JSON_THROW_ON_ERROR));
4250
}
4351
}

‎lib/private/Metadata/FileMetadataMapper.php‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ public function findForGroupForFiles(array $fileIds, string $groupName): array {
8989
continue;
9090
}
9191
$empty = new FileMetadata();
92-
$empty->setMetadata([]);
92+
$empty->setValue('');
9393
$empty->setGroupName($groupName);
9494
$empty->setId($id);
9595
$metadata[$id] = $empty;
@@ -132,13 +132,13 @@ public function update(Entity $entity): Entity {
132132

133133
$idType = $this->getParameterTypeForProperty($entity, 'id');
134134
$groupNameType = $this->getParameterTypeForProperty($entity, 'groupName');
135-
$metadataValue = $entity->getMetadata();
136-
$metadataType = $this->getParameterTypeForProperty($entity, 'metadata');
135+
$value = $entity->getValue();
136+
$valueType = $this->getParameterTypeForProperty($entity, 'value');
137137

138138
$qb = $this->db->getQueryBuilder();
139139

140140
$qb->update($this->tableName)
141-
->set('metadata', $qb->createNamedParameter($metadataValue, $metadataType))
141+
->set('value', $qb->createNamedParameter($value, $valueType))
142142
->where($qb->expr()->eq('id', $qb->createNamedParameter($id, $idType)))
143143
->andWhere($qb->expr()->eq('group_name', $qb->createNamedParameter($groupName, $groupNameType)))
144144
->executeStatement();

‎lib/private/Metadata/MetadataManager.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ public function clearMetadata(int $fileId): void {
7878
$this->fileMetadataMapper->clear($fileId);
7979
}
8080

81+
/**
82+
* @return array<int, FileMetadata>
83+
*/
8184
public function fetchMetadataFor(string $group, array $fileIds): array {
8285
return $this->fileMetadataMapper->findForGroupForFiles($fileIds, $group);
8386
}

‎lib/private/Metadata/Provider/ExifProvider.php‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,12 @@ public function execute(File $file): array {
6565
$size = new FileMetadata();
6666
$size->setGroupName('size');
6767
$size->setId($file->getId());
68-
$size->setMetadata([]);
68+
$size->setObjectAsValue([]);
6969

7070
if (!$data) {
7171
$sizeResult = getimagesizefromstring($file->getContent());
7272
if ($sizeResult !== false) {
73-
$size->setMetadata([
73+
$size->setObjectAsValue([
7474
'width' => $sizeResult[0],
7575
'height' => $sizeResult[1],
7676
]);
@@ -79,7 +79,7 @@ public function execute(File $file): array {
7979
}
8080
} elseif (array_key_exists('COMPUTED', $data)) {
8181
if (array_key_exists('Width', $data['COMPUTED']) && array_key_exists('Height', $data['COMPUTED'])) {
82-
$size->setMetadata([
82+
$size->setObjectAsValue([
8383
'width' => $data['COMPUTED']['Width'],
8484
'height' => $data['COMPUTED']['Height'],
8585
]);
@@ -95,7 +95,7 @@ public function execute(File $file): array {
9595
$gps = new FileMetadata();
9696
$gps->setGroupName('gps');
9797
$gps->setId($file->getId());
98-
$gps->setMetadata([
98+
$gps->setObjectAsValue([
9999
'latitude' => $this->gpsDegreesToDecimal($data['GPS']['GPSLatitude'], $data['GPS']['GPSLatitudeRef']),
100100
'longitude' => $this->gpsDegreesToDecimal($data['GPS']['GPSLongitude'], $data['GPS']['GPSLongitudeRef']),
101101
]);

‎tests/lib/Metadata/FileMetadataMapperTest.php‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,34 +51,34 @@ public function testFindForGroupForFiles() {
5151
$file1 = new FileMetadata();
5252
$file1->setId(1);
5353
$file1->setGroupName('size');
54-
$file1->setMetadata([]);
54+
$file1->setObjectAsValue([]);
5555

5656
$file2 = new FileMetadata();
5757
$file2->setId(2);
5858
$file2->setGroupName('size');
59-
$file2->setMetadata(['width' => 293, 'height' => 23]);
59+
$file2->setObjectAsValue(['width' => 293, 'height' => 23]);
6060

6161
// not added, it's the default
6262
$file3 = new FileMetadata();
6363
$file3->setId(3);
6464
$file3->setGroupName('size');
65-
$file3->setMetadata([]);
65+
$file3->setObjectAsValue([]);
6666

6767
$file4 = new FileMetadata();
6868
$file4->setId(4);
6969
$file4->setGroupName('size');
70-
$file4->setMetadata(['complex' => ["yes", "maybe" => 34.0]]);
70+
$file4->setObjectAsValue(['complex' => ["yes", "maybe" => 34.0]]);
7171

7272
$this->mapper->insert($file1);
7373
$this->mapper->insert($file2);
7474
$this->mapper->insert($file4);
7575

7676
$files = $this->mapper->findForGroupForFiles([1, 2, 3, 4], 'size');
7777

78-
$this->assertEquals($files[1]->getMetadata(), $file1->getMetadata());
79-
$this->assertEquals($files[2]->getMetadata(), $file2->getMetadata());
80-
$this->assertEquals($files[3]->getMetadata(), $file3->getMetadata());
81-
$this->assertEquals($files[4]->getMetadata(), $file4->getMetadata());
78+
$this->assertEquals($files[1]->getValue(), $file1->getValue());
79+
$this->assertEquals($files[2]->getValue(), $file2->getValue());
80+
$this->assertEquals($files[3]->getValue(), $file3->getValue());
81+
$this->assertEquals($files[4]->getValue(), $file4->getValue());
8282

8383
$this->mapper->clear(1);
8484
$this->mapper->clear(2);

0 commit comments

Comments
 (0)