Skip to content

Commit 3481821

Browse files
artongeartonge
committed
Migrate metadata as JSON to value as STRING
Signed-off-by: Louis Chemineau <louis@chmn.me>
1 parent 691aa8d commit 3481821

8 files changed

Lines changed: 208 additions & 15 deletions

File tree

‎apps/dav/lib/Connector/Sabre/FilesPlugin.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -436,7 +436,7 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node)
436436
\OC::$server->get(LoggerInterface::class)->debug('Inefficient fetching of metadata');
437437
}
438438

439-
return json_encode((object)$sizeMetadata->getMetadata(), JSON_THROW_ON_ERROR);
439+
return $sizeMetadata->getValue();
440440
});
441441
}
442442
}

‎core/Migrations/Version24000Date20220404230027.php‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,14 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
5252
'notnull' => true,
5353
'length' => 50,
5454
]);
55-
$table->addColumn('metadata', Types::JSON, [
55+
$table->addColumn('value', Types::STRING, [
5656
'notnull' => true,
5757
]);
5858
$table->setPrimaryKey(['id', 'group_name'], 'file_metadata_idx');
59+
60+
return $schema;
5961
}
60-
return $schema;
62+
63+
return null;
6164
}
6265
}

‎core/Migrations/Version27000Date20230309104325.php‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* @copyright Copyright (c) 2023 Louis Chmn <louis@chmn.me>
7+
*
8+
* @author Louis Chmn <louis@chmn.me>
9+
*
10+
* @license GNU AGPL version 3 or any later version
11+
*
12+
* This program is free software: you can redistribute it and/or modify
13+
* it under the terms of the GNU Affero General Public License as
14+
* published by the Free Software Foundation, either version 3 of the
15+
* License, or (at your option) any later version.
16+
*
17+
* This program is distributed in the hope that it will be useful,
18+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
19+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20+
* GNU Affero General Public License for more details.
21+
*
22+
* You should have received a copy of the GNU Affero General Public License
23+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
24+
*
25+
*/
26+
27+
namespace OC\Core\Migrations;
28+
29+
use Closure;
30+
use OCP\DB\ISchemaWrapper;
31+
use OCP\DB\QueryBuilder\IQueryBuilder;
32+
use OCP\DB\Types;
33+
use OCP\IDBConnection;
34+
use OCP\Migration\IOutput;
35+
use OCP\Migration\SimpleMigrationStep;
36+
37+
/**
38+
* Migrate oc_file_metadata.metadata as JSON type to oc_file_metadata.value a STRING type
39+
* @see \OC\Metadata\FileMetadata
40+
*/
41+
class Version27000Date20230309104325 extends SimpleMigrationStep {
42+
public function __construct(
43+
private IDBConnection $connection
44+
) {
45+
}
46+
47+
/**
48+
* @param IOutput $output
49+
* @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
50+
* @param array $options
51+
* @return null|ISchemaWrapper
52+
*/
53+
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
54+
/** @var ISchemaWrapper $schema */
55+
$schema = $schemaClosure();
56+
$metadataTable = $schema->getTable('file_metadata');
57+
58+
if ($metadataTable->hasColumn('value')) {
59+
return null;
60+
}
61+
62+
$metadataTable->addColumn('value', Types::STRING, [
63+
'notnull' => true,
64+
]);
65+
return $schema;
66+
}
67+
68+
69+
/**
70+
* @param IOutput $output
71+
* @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
72+
* @param array $options
73+
* @return void
74+
*/
75+
public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options) {
76+
/** @var ISchemaWrapper $schema */
77+
$schema = $schemaClosure();
78+
$metadataTable = $schema->getTable('file_metadata');
79+
80+
if (!$metadataTable->hasColumn('metadata')) {
81+
return;
82+
}
83+
84+
$insertQuery = $this->connection->getQueryBuilder();
85+
$insertQuery->insert('file_metadata')
86+
->where($insertQuery->expr()->eq('id', $insertQuery->createParameter('id')))
87+
->values(['value' => $insertQuery->createParameter('value')]);
88+
89+
$selectQuery = $this->connection->getQueryBuilder();
90+
$selectQuery->select('id, metadata')
91+
->from('file_metadata')
92+
->orderBy('id', 'ASC')
93+
->setMaxResults(1000);
94+
95+
$offset = 0;
96+
$movedRows = 0;
97+
do {
98+
$movedRows = $this->chunkedCopying($insertQuery, $selectQuery, $offset);
99+
$offset += $movedRows;
100+
} while ($movedRows !== 0);
101+
}
102+
103+
protected function chunkedCopying(IQueryBuilder $insertQuery, IQueryBuilder $selectQuery, int $offset): int {
104+
$this->connection->beginTransaction();
105+
106+
$results = $selectQuery
107+
->setFirstResult($offset)
108+
->executeQuery();
109+
110+
while ($row = $results->fetch()) {
111+
$insertQuery
112+
->setParameter('id', (int)$row['id'])
113+
->setParameter('value', (int)$row['metadata'])
114+
->executeStatement();
115+
}
116+
117+
$results->closeCursor();
118+
$this->connection->commit();
119+
120+
return $results->rowCount();
121+
}
122+
}

‎core/Migrations/Version27000Date20230309104802.php‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* @copyright Copyright (c) 2023 Louis Chmn <louis@chmn.me>
7+
*
8+
* @author Louis Chmn <louis@chmn.me>
9+
*
10+
* @license GNU AGPL version 3 or any later version
11+
*
12+
* This program is free software: you can redistribute it and/or modify
13+
* it under the terms of the GNU Affero General Public License as
14+
* published by the Free Software Foundation, either version 3 of the
15+
* License, or (at your option) any later version.
16+
*
17+
* This program is distributed in the hope that it will be useful,
18+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
19+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20+
* GNU Affero General Public License for more details.
21+
*
22+
* You should have received a copy of the GNU Affero General Public License
23+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
24+
*
25+
*/
26+
27+
namespace OC\Core\Migrations;
28+
29+
use Closure;
30+
use OCP\DB\ISchemaWrapper;
31+
use OCP\Migration\IOutput;
32+
use OCP\Migration\SimpleMigrationStep;
33+
34+
/**
35+
* Migrate oc_file_metadata.metadata as JSON type to oc_file_metadata.value a STRING type
36+
* @see \OC\Metadata\FileMetadata
37+
*/
38+
class Version27000Date20230309104802 extends SimpleMigrationStep {
39+
/**
40+
* @param IOutput $output
41+
* @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
42+
* @param array $options
43+
* @return null|ISchemaWrapper
44+
*/
45+
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
46+
/** @var ISchemaWrapper $schema */
47+
$schema = $schemaClosure();
48+
$metadataTable = $schema->getTable('file_metadata');
49+
50+
if ($metadataTable->hasColumn('metadata')) {
51+
$metadataTable->dropColumn('metadata');
52+
return $schema;
53+
}
54+
55+
return null;
56+
}
57+
}

‎lib/private/Metadata/FileMetadata.php‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,24 @@
2828
/**
2929
* @method string getGroupName()
3030
* @method void setGroupName(string $groupName)
31-
* @method array getMetadata()
32-
* @method void setMetadata(array $metadata)
31+
* @method string getValue()
32+
* @method void setValue(string $value)
3333
* @see \OC\Core\Migrations\Version240000Date20220404230027
3434
*/
3535
class FileMetadata extends Entity {
3636
protected ?string $groupName = null;
37-
protected ?array $metadata = null;
37+
protected ?string $value = null;
3838

3939
public function __construct() {
4040
$this->addType('groupName', 'string');
41-
$this->addType('metadata', Types::JSON);
41+
$this->addType('value', Types::STRING);
42+
}
43+
44+
public function getDecodedValue(): object {
45+
return json_decode($this->getValue());
46+
}
47+
48+
public function setObjectAsValue(array $value): void {
49+
$this->setValue(json_encode($value, JSON_THROW_ON_ERROR));
4250
}
4351
}

‎lib/private/Metadata/FileMetadataMapper.php‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ public function findForGroupForFiles(array $fileIds, string $groupName): array {
8989
continue;
9090
}
9191
$empty = new FileMetadata();
92-
$empty->setMetadata([]);
92+
$empty->setValue('');
9393
$empty->setGroupName($groupName);
9494
$empty->setId($id);
9595
$metadata[$id] = $empty;
@@ -132,13 +132,13 @@ public function update(Entity $entity): Entity {
132132

133133
$idType = $this->getParameterTypeForProperty($entity, 'id');
134134
$groupNameType = $this->getParameterTypeForProperty($entity, 'groupName');
135-
$metadataValue = $entity->getMetadata();
136-
$metadataType = $this->getParameterTypeForProperty($entity, 'metadata');
135+
$value = $entity->getValue();
136+
$valueType = $this->getParameterTypeForProperty($entity, 'value');
137137

138138
$qb = $this->db->getQueryBuilder();
139139

140140
$qb->update($this->tableName)
141-
->set('metadata', $qb->createNamedParameter($metadataValue, $metadataType))
141+
->set('value', $qb->createNamedParameter($value, $valueType))
142142
->where($qb->expr()->eq('id', $qb->createNamedParameter($id, $idType)))
143143
->andWhere($qb->expr()->eq('group_name', $qb->createNamedParameter($groupName, $groupNameType)))
144144
->executeStatement();

‎lib/private/Metadata/MetadataManager.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ public function clearMetadata(int $fileId): void {
7878
$this->fileMetadataMapper->clear($fileId);
7979
}
8080

81+
/**
82+
* @return array<int, FileMetadata>
83+
*/
8184
public function fetchMetadataFor(string $group, array $fileIds): array {
8285
return $this->fileMetadataMapper->findForGroupForFiles($fileIds, $group);
8386
}

‎lib/private/Metadata/Provider/ExifProvider.php‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,12 @@ public function execute(File $file): array {
6565
$size = new FileMetadata();
6666
$size->setGroupName('size');
6767
$size->setId($file->getId());
68-
$size->setMetadata([]);
68+
$size->setObjectAsValue([]);
6969

7070
if (!$data) {
7171
$sizeResult = getimagesizefromstring($file->getContent());
7272
if ($sizeResult !== false) {
73-
$size->setMetadata([
73+
$size->setObjectAsValue([
7474
'width' => $sizeResult[0],
7575
'height' => $sizeResult[1],
7676
]);
@@ -79,7 +79,7 @@ public function execute(File $file): array {
7979
}
8080
} elseif (array_key_exists('COMPUTED', $data)) {
8181
if (array_key_exists('Width', $data['COMPUTED']) && array_key_exists('Height', $data['COMPUTED'])) {
82-
$size->setMetadata([
82+
$size->setObjectAsValue([
8383
'width' => $data['COMPUTED']['Width'],
8484
'height' => $data['COMPUTED']['Height'],
8585
]);
@@ -95,7 +95,7 @@ public function execute(File $file): array {
9595
$gps = new FileMetadata();
9696
$gps->setGroupName('gps');
9797
$gps->setId($file->getId());
98-
$gps->setMetadata([
98+
$gps->setObjectAsValue([
9999
'latitude' => $this->gpsDegreesToDecimal($data['GPS']['GPSLatitude'], $data['GPS']['GPSLatitudeRef']),
100100
'longitude' => $this->gpsDegreesToDecimal($data['GPS']['GPSLongitude'], $data['GPS']['GPSLongitudeRef']),
101101
]);

0 commit comments

Comments
 (0)