1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
|
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import path from 'node:path';
import { fdir } from 'fdir';
import { Ignore } from './ignore.js';
import * as cache from './crawlCache.js';
export interface CrawlOptions {
// The directory to start the crawl from.
crawlDirectory: string;
// The project's root directory, for path relativity.
cwd: string;
// The fdir maxDepth option.
maxDepth?: number;
// A pre-configured Ignore instance.
ignore: Ignore;
// Caching options.
cache: boolean;
cacheTtl: number;
}
function toPosixPath(p: string) {
return p.split(path.sep).join(path.posix.sep);
}
export async function crawl(options: CrawlOptions): Promise<string[]> {
if (options.cache) {
const cacheKey = cache.getCacheKey(
options.crawlDirectory,
options.ignore.getFingerprint(),
options.maxDepth,
);
const cachedResults = cache.read(cacheKey);
if (cachedResults) {
return cachedResults;
}
}
const posixCwd = toPosixPath(options.cwd);
const posixCrawlDirectory = toPosixPath(options.crawlDirectory);
let results: string[];
try {
const dirFilter = options.ignore.getDirectoryFilter();
const api = new fdir()
.withRelativePaths()
.withDirs()
.withPathSeparator('/') // Always use unix style paths
.exclude((_, dirPath) => {
const relativePath = path.posix.relative(posixCrawlDirectory, dirPath);
return dirFilter(`${relativePath}/`);
});
if (options.maxDepth !== undefined) {
api.withMaxDepth(options.maxDepth);
}
results = await api.crawl(options.crawlDirectory).withPromise();
} catch (_e) {
// The directory probably doesn't exist.
return [];
}
const relativeToCrawlDir = path.posix.relative(posixCwd, posixCrawlDirectory);
const relativeToCwdResults = results.map((p) =>
path.posix.join(relativeToCrawlDir, p),
);
if (options.cache) {
const cacheKey = cache.getCacheKey(
options.crawlDirectory,
options.ignore.getFingerprint(),
options.maxDepth,
);
cache.write(cacheKey, relativeToCwdResults, options.cacheTtl * 1000);
}
return relativeToCwdResults;
}
|
