Skip to content

Commit 353ed85

Browse files
rullzerrullzer
committed
Move to subfolders for preview files
Else the number of files can grow very large very quickly in the preview folder. Esp on large systems. This generates the md5 of the fileid. And then creates folders of the first 7 charts. In that folder is then a folder with the fileid. And inside there are the previews. Signed-off-by: Roeland Jago Douma <roeland@famdouma.nl>
1 parent a307d82 commit 353ed85

7 files changed

Lines changed: 221 additions & 43 deletions

File tree

‎lib/composer/composer/autoload_classmap.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1151,6 +1151,7 @@
11511151
'OC\\Preview\\ProviderV2' => $baseDir . '/lib/private/Preview/ProviderV2.php',
11521152
'OC\\Preview\\SVG' => $baseDir . '/lib/private/Preview/SVG.php',
11531153
'OC\\Preview\\StarOffice' => $baseDir . '/lib/private/Preview/StarOffice.php',
1154+
'OC\\Preview\\Storage\\Root' => $baseDir . '/lib/private/Preview/Storage/Root.php',
11541155
'OC\\Preview\\TIFF' => $baseDir . '/lib/private/Preview/TIFF.php',
11551156
'OC\\Preview\\TXT' => $baseDir . '/lib/private/Preview/TXT.php',
11561157
'OC\\Preview\\Watcher' => $baseDir . '/lib/private/Preview/Watcher.php',

‎lib/composer/composer/autoload_static.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1180,6 +1180,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
11801180
'OC\\Preview\\ProviderV2' => __DIR__ . '/../../..' . '/lib/private/Preview/ProviderV2.php',
11811181
'OC\\Preview\\SVG' => __DIR__ . '/../../..' . '/lib/private/Preview/SVG.php',
11821182
'OC\\Preview\\StarOffice' => __DIR__ . '/../../..' . '/lib/private/Preview/StarOffice.php',
1183+
'OC\\Preview\\Storage\\Root' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/Root.php',
11831184
'OC\\Preview\\TIFF' => __DIR__ . '/../../..' . '/lib/private/Preview/TIFF.php',
11841185
'OC\\Preview\\TXT' => __DIR__ . '/../../..' . '/lib/private/Preview/TXT.php',
11851186
'OC\\Preview\\Watcher' => __DIR__ . '/../../..' . '/lib/private/Preview/Watcher.php',

‎lib/private/Preview/BackgroundCleanupJob.php‎

Lines changed: 74 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -28,38 +28,60 @@
2828

2929
use OC\BackgroundJob\TimedJob;
3030
use OC\Files\AppData\Factory;
31+
use OC\Preview\Storage\Root;
3132
use OCP\DB\QueryBuilder\IQueryBuilder;
33+
use OCP\Files\IMimeTypeLoader;
3234
use OCP\Files\NotFoundException;
3335
use OCP\Files\NotPermittedException;
36+
use OCP\Files\SimpleFS\ISimpleFolder;
3437
use OCP\IDBConnection;
3538

3639
class BackgroundCleanupJob extends TimedJob {
3740

3841
/** @var IDBConnection */
3942
private $connection;
4043

41-
/** @var Factory */
42-
private $appDataFactory;
44+
/** @var Root */
45+
private $previewFolder;
4346

4447
/** @var bool */
4548
private $isCLI;
4649

50+
/** @var IMimeTypeLoader */
51+
private $mimeTypeLoader;
52+
4753
public function __construct(IDBConnection $connection,
48-
Factory $appDataFactory,
54+
Root $previewFolder,
55+
IMimeTypeLoader $mimeTypeLoader,
4956
bool $isCLI) {
5057
// Run at most once an hour
5158
$this->setInterval(3600);
5259

5360
$this->connection = $connection;
54-
$this->appDataFactory = $appDataFactory;
61+
$this->previewFolder = $previewFolder;
5562
$this->isCLI = $isCLI;
63+
$this->mimeTypeLoader = $mimeTypeLoader;
5664
}
5765

5866
public function run($argument) {
59-
$previews = $this->appDataFactory->get('preview');
67+
foreach ($this->getDeletedFiles() as $fileId) {
68+
try {
69+
$preview = $this->previewFolder->getFolder((string)$fileId);
70+
$preview->delete();
71+
} catch (NotFoundException $e) {
72+
// continue
73+
} catch (NotPermittedException $e) {
74+
// continue
75+
}
76+
}
77+
}
6078

61-
$previewFodlerId = $previews->getId();
79+
private function getDeletedFiles(): \Iterator {
80+
yield from $this->getOldPreviewLocations();
81+
yield from $this->getNewPreviewLocations();
82+
}
6283

84+
private function getOldPreviewLocations(): \Iterator {
6385
$qb = $this->connection->getQueryBuilder();
6486
$qb->select('a.name')
6587
->from('filecache', 'a')
@@ -69,7 +91,9 @@ public function run($argument) {
6991
->where(
7092
$qb->expr()->isNull('b.fileid')
7193
)->andWhere(
72-
$qb->expr()->eq('a.parent', $qb->createNamedParameter($previewFodlerId))
94+
$qb->expr()->eq('a.parent', $qb->createNamedParameter($this->previewFolder->getId()))
95+
)->andWhere(
96+
$qb->expr()->like('a.name', $qb->createNamedParameter('__%'))
7397
);
7498

7599
if (!$this->isCLI) {
@@ -79,14 +103,49 @@ public function run($argument) {
79103
$cursor = $qb->execute();
80104

81105
while ($row = $cursor->fetch()) {
82-
try {
83-
$preview = $previews->getFolder($row['name']);
84-
$preview->delete();
85-
} catch (NotFoundException $e) {
86-
// continue
87-
} catch (NotPermittedException $e) {
88-
// continue
89-
}
106+
yield $row['name'];
107+
}
108+
109+
$cursor->closeCursor();
110+
}
111+
112+
private function getNewPreviewLocations(): \Iterator {
113+
$qb = $this->connection->getQueryBuilder();
114+
$qb->select('path', 'mimetype')
115+
->from('filecache')
116+
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId())));
117+
$cursor = $qb->execute();
118+
$data = $cursor->fetch();
119+
$cursor->closeCursor();
120+
121+
if ($data === null) {
122+
return [];
123+
}
124+
125+
$like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%';
126+
127+
$qb = $this->connection->getQueryBuilder();
128+
$qb->select('a.name')
129+
->from('filecache', 'a')
130+
->leftJoin('a', 'filecache', 'b', $qb->expr()->eq(
131+
$qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid'
132+
))
133+
->where(
134+
$qb->expr()->andX(
135+
$qb->expr()->isNull('b.fileid'),
136+
$qb->expr()->like('a.path', $qb->createNamedParameter($like)),
137+
$qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory')))
138+
)
139+
);
140+
141+
if (!$this->isCLI) {
142+
$qb->setMaxResults(10);
143+
}
144+
145+
$cursor = $qb->execute();
146+
147+
while ($row = $cursor->fetch()) {
148+
yield $row['name'];
90149
}
91150

92151
$cursor->closeCursor();

‎lib/private/Preview/Storage/Root.php‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
<?php
2+
declare(strict_types=1);
3+
/**
4+
* @copyright Copyright (c) 2020, Roeland Jago Douma <roeland@famdouma.nl>
5+
*
6+
* @author Roeland Jago Douma <roeland@famdouma.nl>
7+
*
8+
* @license GNU AGPL version 3 or any later version
9+
*
10+
* This program is free software: you can redistribute it and/or modify
11+
* it under the terms of the GNU Affero General Public License as
12+
* published by the Free Software Foundation, either version 3 of the
13+
* License, or (at your option) any later version.
14+
*
15+
* This program is distributed in the hope that it will be useful,
16+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
17+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18+
* GNU Affero General Public License for more details.
19+
*
20+
* You should have received a copy of the GNU Affero General Public License
21+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
22+
*
23+
*/
24+
25+
namespace OC\Preview\Storage;
26+
27+
use OC\Files\AppData\AppData;
28+
use OC\SystemConfig;
29+
use OCP\Files\IRootFolder;
30+
use OCP\Files\NotFoundException;
31+
use OCP\Files\SimpleFS\ISimpleFolder;
32+
33+
class Root extends AppData {
34+
35+
public function __construct(IRootFolder $rootFolder, SystemConfig $systemConfig) {
36+
parent::__construct($rootFolder, $systemConfig, 'preview');
37+
}
38+
39+
40+
public function getFolder(string $name): ISimpleFolder {
41+
$internalFolder = $this->getInternalFolder($name);
42+
43+
try {
44+
return parent::getFolder($internalFolder);
45+
} catch (NotFoundException $e) {
46+
/*
47+
* The new folder structure is not found.
48+
* Lets try the old one
49+
*/
50+
}
51+
52+
return parent::getFolder($name);
53+
}
54+
55+
public function newFolder(string $name): ISimpleFolder {
56+
$internalFolder = $this->getInternalFolder($name);
57+
return parent::newFolder($internalFolder);
58+
}
59+
60+
/*
61+
* Do not allow directory listing on this special root
62+
* since it gets to big and time consuming
63+
*/
64+
public function getDirectoryListing(): array {
65+
return [];
66+
}
67+
68+
private function getInternalFolder(string $name): string {
69+
return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name;
70+
}
71+
}

‎lib/private/Server.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ public function __construct($webRoot, \OC\Config $config) {
281281
return new PreviewManager(
282282
$c->getConfig(),
283283
$c->getRootFolder(),
284-
$c->getAppDataDir('preview'),
284+
new \OC\Preview\Storage\Root($c->getRootFolder(), $c->getSystemConfig(), 'preview'),
285285
$c->getEventDispatcher(),
286286
$c->getGeneratorHelper(),
287287
$c->getSession()->get('user_id')
@@ -291,7 +291,7 @@ public function __construct($webRoot, \OC\Config $config) {
291291

292292
$this->registerService(\OC\Preview\Watcher::class, function (Server $c) {
293293
return new \OC\Preview\Watcher(
294-
$c->getAppDataDir('preview')
294+
new \OC\Preview\Storage\Root($c->getRootFolder(), $c->getSystemConfig(), 'preview')
295295
);
296296
});
297297

‎lib/public/DB/QueryBuilder/IQueryBuilder.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,7 @@ public function leftJoin($fromAlias, $join, $alias, $condition = null);
498498
* ->select('u.name')
499499
* ->from('users', 'u')
500500
* ->rightJoin('u', 'phonenumbers', 'p', 'p.is_primary = 1');
501-
* </code>
501+
* </code>/
502502
*
503503
* @param string $fromAlias The alias that points to a from clause.
504504
* @param string $join The table name to join.

0 commit comments

Comments
 (0)