26 std::function<
void(std::string_view)> reportProgress_)
27 : fileCache(
std::move(fileCache_))
28 , getDirectories(
std::move(getDirectories_))
29 , reportProgress(
std::move(reportProgress_))
45void FilePoolCore::insert(
const Sha1Sum&
sum, time_t time,
const std::string& filename)
47 stringBuffer.push_back(filename);
48 auto idx = pool.
emplace(
sum, time, stringBuffer.back()).idx;
50 sha1Index.insert(it, idx);
55FilePoolCore::Sha1Index::iterator FilePoolCore::getSha1Iterator(Index idx, Entry& entry)
63 assert(
false);
return sha1Index.end();
66void FilePoolCore::remove(Sha1Index::iterator it)
69 filenameIndex.
erase(idx);
75void FilePoolCore::remove(Index idx, Entry& entry)
77 remove(getSha1Iterator(idx, entry));
80void FilePoolCore::remove(Index idx)
82 remove(idx, pool[idx]);
91bool FilePoolCore::adjustSha1(Sha1Index::iterator it, Entry& entry,
const Sha1Sum& newSum)
103 rotate(newIt, it, it + 1);
113bool FilePoolCore::adjustSha1(Index idx, Entry& entry,
const Sha1Sum& newSum)
115 return adjustSha1(getSha1Iterator(idx, entry), entry, newSum);
118time_t FilePoolCore::Entry::getTime()
126void FilePoolCore::Entry::setTime(time_t
t)
133static std::optional<std::tuple<Sha1Sum, const char*, std::string_view>> parse(
134 std::span<char> line)
136 if (line.size() <= 68)
return {};
139 if (line[40] !=
' ')
return {};
140 if (line[41] !=
' ')
return {};
141 if (line[45] !=
' ')
return {};
142 if (line[49] !=
' ')
return {};
143 if (line[52] !=
' ')
return {};
144 if (line[55] !=
':')
return {};
145 if (line[58] !=
':')
return {};
146 if (line[61] !=
' ')
return {};
147 if (line[66] !=
' ')
return {};
148 if (line[67] !=
' ')
return {};
150 Sha1Sum sha1(Sha1Sum::UninitializedTag{});
152 sha1.parse40(subspan<40>(line));
153 }
catch (MSXException&) {
157 const char* timeStr = &line[42];
160 std::string_view filename(&line[68], line.size() - 68);
162 return std::tuple{sha1, timeStr, filename};
165void FilePoolCore::readSha1sums()
167 assert(sha1Index.empty());
168 assert(fileMem.
empty());
170 File file(fileCache);
171 auto size = file.getSize();
173 file.read(std::span{fileMem.
data(),
size});
174 fileMem[
size] =
'\n';
178 char* data = fileMem.
data();
179 char* data_end = data +
size + 1;
180 while (data != data_end) {
182 char* it =
static_cast<char*
>(memchr(data,
'\n', data_end - data));
183 if (it ==
nullptr) it = data_end;
184 if ((it != data) && (it[-1] ==
'\r')) --it;
186 if (
auto r = parse({data, it})) {
187 auto [
sum, timeStr, filename] = *r;
188 sha1Index.push_back(pool.
emplace(
sum, timeStr, filename).idx);
193 return c !=
one_of(
'\n',
'\r');
206 auto n = sha1Index.size();
210 Index idx = sha1Index[n];
211 bool inserted = filenameIndex.
insert(idx);
220void FilePoolCore::writeSha1sums()
224 if (!file.is_open()) {
227 for (
auto idx : sha1Index) {
228 const auto& entry = pool[idx];
229 file << entry.sum.toString() <<
" ";
231 file << entry.timeStr;
236 file <<
" " << entry.filename <<
'\n';
242 File result = getFromPool(sha1sum);
243 if (result.
is_open())
return result;
247 ScanProgress progress;
249 progress.amountScanned = 0;
251 for (
auto& [path, types] : getDirectories()) {
254 if (result.
is_open())
return result;
265 constexpr size_t STEP_SIZE = 1024 * 1024;
267 auto data = file.
mmap();
270 size_t size = data.size();
272 size_t remaining =
size;
274 bool everShowedProgress =
false;
276 auto report = [&](
size_t percentage) {
278 "... ", percentage,
'%'));
281 while (remaining > STEP_SIZE) {
282 sha1.
update({&data[done], STEP_SIZE});
284 remaining -= STEP_SIZE;
287 if ((now - lastShowedProgress) > 250'000) {
288 report((100 * done) /
size);
289 lastShowedProgress = now;
290 everShowedProgress =
true;
295 sha1.
update({&data[done], remaining});
297 if (everShowedProgress) {
303File FilePoolCore::getFromPool(
const Sha1Sum& sha1sum)
310 auto it =
begin(sha1Index) + i;
311 auto& entry = pool[*it];
320 File file(std::string(entry.filename));
322 if (entry.getTime() == newTime) {
328 entry.setTime(newTime);
330 auto newSum = calcSha1sum(file);
331 if (newSum == sha1sum) {
338 if (adjustSha1(it, entry, newSum)) {
345 }
catch (FileException&) {
355File FilePoolCore::scanDirectory(
356 const Sha1Sum& sha1sum,
const std::string& directory, std::string_view poolPath,
357 ScanProgress& progress)
365 assert(!result.is_open());
368 result = scanFile(sha1sum, path, st, poolPath, progress);
369 return !result.is_open();
375File FilePoolCore::scanFile(
const Sha1Sum& sha1sum,
const std::string& filename,
377 ScanProgress& progress)
379 ++progress.amountScanned;
382 if (now > (progress.lastTime + 250'000)) {
383 progress.lastTime = now;
385 "Searching for file with sha1sum ", sha1sum.toString(),
386 "...\nIndexing filepool ", poolPath,
": [",
387 progress.amountScanned,
"]: ",
388 std::string_view(filename).substr(poolPath.size())));
392 if (
auto [idx, entry] = findInDatabase(filename); idx == Index(-1)) {
396 auto sum = calcSha1sum(file);
397 insert(
sum, time, filename);
398 if (
sum == sha1sum) {
401 }
catch (FileException&) {
406 assert(filename == entry->filename);
408 if (entry->getTime() == time) {
410 if (entry->sum == sha1sum) {
411 return File(filename);
416 auto sum = calcSha1sum(file);
417 entry->setTime(time);
418 adjustSha1(idx, *entry,
sum);
419 if (
sum == sha1sum) {
423 }
catch (FileException&) {
431std::pair<FilePoolCore::Index, FilePoolCore::Entry*> FilePoolCore::findInDatabase(std::string_view filename)
433 auto it = filenameIndex.
find(filename);
434 if (!it)
return {Index(-1),
nullptr};
437 auto& entry = pool[idx];
438 assert(entry.filename == filename);
442 return {Index(-1),
nullptr};
444 return {idx, &entry};
450 const std::string& filename = file.
getURL();
452 auto [idx, entry] = findInDatabase(filename);
453 if (idx != Index(-1)) {
454 if (entry->getTime() == time) {
462 auto sum = calcSha1sum(file);
463 if (idx == Index(-1)) {
465 insert(
sum, time, filename);
468 entry->setTime(time);
469 adjustSha1(idx, *entry,
sum);
EmplaceResult emplace(Args &&...args)
bool insert(Value2 &&val)
Value * find(const Value2 &val) const
bool erase(const Value2 &val)
Sha1Sum getSha1Sum(File &file)
Calculate sha1sum for the given File object.
FilePoolCore(std::string fileCache, std::function< Directories()> getDirectories, std::function< void(std::string_view)> reportProgress)
File getFile(FileType fileType, const Sha1Sum &sha1sum)
Search file with the given sha1sum.
std::vector< Dir > Directories
std::span< const uint8_t > mmap()
Map file in memory.
std::string_view getOriginalName()
Get Original filename for this object.
time_t getModificationDate()
Get the date/time of last modification.
bool is_open() const
Return true iff this file handle refers to an open file.
const std::string & getURL() const
Returns the URL of this file object.
void resize(size_t size)
Grow or shrink the memory block.
bool empty() const
No memory allocated?
const T * data() const
Returns pointer to the start of the memory buffer.
Helper class to perform a sha1 calculation.
Sha1Sum digest()
Get the final hash.
void update(std::span< const uint8_t > data)
Incrementally calculate the hash value.
This class represents the result of a sha1 calculation (a 160-bit value).
mat4 rotate(float angle, const vec3 &axis)
std::string toString(time_t time)
time_t fromString(std::span< const char, 24 > s)
constexpr time_t INVALID_TIME_T
string expandTilde(string path)
Expand the '~' character to the users home directory.
void openOfStream(std::ofstream &stream, zstring_view filename)
Open an ofstream in a platform-independent manner.
time_t getModificationDate(const Stat &st)
Get the date/time of last modification.
uint64_t getTime()
Get current (real) time in us.
This file implemented 3 utility functions:
bool foreach_file_recursive(std::string path, FileAction fileAction)
bool is_sorted(ForwardRange &&range, Compare comp={}, Proj proj={})
auto find_if(InputRange &&range, UnaryPredicate pred)
auto upper_bound(ForwardRange &&range, const T &value, Compare comp={}, Proj proj={})
auto equal_range(ForwardRange &&range, const T &value, Compare comp={})
constexpr void sort(RandomAccessRange &&range)
size_t size(std::string_view utf8)
auto distance(octet_iterator first, octet_iterator last)
auto sum(InputRange &&range, Proj proj={})
TemporaryString tmpStrCat(Ts &&... ts)
const Sha1Sum & operator()(FilePoolCore::Index idx) const
const FilePoolCore::Pool & pool
constexpr auto begin(const zstring_view &x)