26 std::function<
void(std::string_view,
float)> reportProgress_)
27 : fileCache(
std::move(fileCache_))
28 , getDirectories(
std::move(getDirectories_))
29 , reportProgress(
std::move(reportProgress_))
45void FilePoolCore::insert(
const Sha1Sum&
sum, time_t time,
const std::string& filename)
47 stringBuffer.push_back(filename);
48 auto idx = pool.
emplace(
sum, time, stringBuffer.back()).idx;
50 sha1Index.insert(it, idx);
55FilePoolCore::Sha1Index::iterator FilePoolCore::getSha1Iterator(Index idx, Entry& entry)
63 assert(
false);
return sha1Index.end();
66void FilePoolCore::remove(Sha1Index::iterator it)
69 filenameIndex.
erase(idx);
75void FilePoolCore::remove(Index idx, Entry& entry)
77 remove(getSha1Iterator(idx, entry));
80void FilePoolCore::remove(Index idx)
82 remove(idx, pool[idx]);
91bool FilePoolCore::adjustSha1(Sha1Index::iterator it, Entry& entry,
const Sha1Sum& newSum)
103 rotate(newIt, it, it + 1);
113bool FilePoolCore::adjustSha1(Index idx, Entry& entry,
const Sha1Sum& newSum)
115 return adjustSha1(getSha1Iterator(idx, entry), entry, newSum);
118time_t FilePoolCore::Entry::getTime()
126void FilePoolCore::Entry::setTime(time_t
t)
133static std::optional<std::tuple<Sha1Sum, const char*, std::string_view>> parse(
134 std::span<char> line)
136 if (line.size() <= 68)
return {};
139 if (line[40] !=
' ')
return {};
140 if (line[41] !=
' ')
return {};
141 if (line[45] !=
' ')
return {};
142 if (line[49] !=
' ')
return {};
143 if (line[52] !=
' ')
return {};
144 if (line[55] !=
':')
return {};
145 if (line[58] !=
':')
return {};
146 if (line[61] !=
' ')
return {};
147 if (line[66] !=
' ')
return {};
148 if (line[67] !=
' ')
return {};
150 Sha1Sum sha1(Sha1Sum::UninitializedTag{});
152 sha1.parse40(subspan<40>(line));
153 }
catch (MSXException&) {
157 const char* timeStr = &line[42];
160 std::string_view filename(&line[68], line.size() - 68);
162 return std::tuple{sha1, timeStr, filename};
165void FilePoolCore::readSha1sums()
167 assert(sha1Index.empty());
168 assert(fileMem.
empty());
170 File file(fileCache);
171 auto size = file.getSize();
173 file.read(std::span{fileMem.
data(),
size});
174 fileMem[
size] =
'\n';
178 char* data = fileMem.
data();
179 char* data_end = data +
size + 1;
180 while (data != data_end) {
182 char* it =
static_cast<char*
>(memchr(data,
'\n', data_end - data));
183 if (it ==
nullptr) it = data_end;
184 if ((it != data) && (it[-1] ==
'\r')) --it;
186 if (
auto r = parse({data, it})) {
187 auto [
sum, timeStr, filename] = *r;
188 sha1Index.push_back(pool.
emplace(
sum, timeStr, filename).idx);
193 return c !=
one_of(
'\n',
'\r');
206 auto n = sha1Index.size();
210 Index idx = sha1Index[n];
211 bool inserted = filenameIndex.
insert(idx);
220void FilePoolCore::writeSha1sums()
224 if (!file.is_open()) {
227 for (
auto idx : sha1Index) {
228 const auto& entry = pool[idx];
229 file << entry.sum.toString() <<
" ";
231 file << entry.timeStr;
236 file <<
" " << entry.filename <<
'\n';
242 File result = getFromPool(sha1sum);
243 if (result.
is_open())
return result;
247 ScanProgress progress {
251 for (
auto& [path, types] : getDirectories()) {
255 if (progress.printed) {
263 if (progress.printed) {
264 reportProgress(
tmpStrCat(
"Did not find file with sha1sum ", sha1sum.
toString()), 1.0f);
273 constexpr size_t STEP_SIZE = 1024 * 1024;
275 auto data = file.
mmap();
278 size_t size = data.size();
280 size_t remaining =
size;
282 bool everShowedProgress =
false;
284 auto report = [&](
float fraction) {
289 while (remaining > STEP_SIZE) {
290 sha1.
update({&data[done], STEP_SIZE});
292 remaining -= STEP_SIZE;
295 if ((now - lastShowedProgress) > 250'000) {
296 report(
float(done) /
float(
size));
297 lastShowedProgress = now;
298 everShowedProgress =
true;
303 sha1.
update({&data[done], remaining});
305 if (everShowedProgress) {
311File FilePoolCore::getFromPool(
const Sha1Sum& sha1sum)
318 auto it =
begin(sha1Index) + i;
319 auto& entry = pool[*it];
328 File file(std::string(entry.filename));
330 if (entry.getTime() == newTime) {
336 entry.setTime(newTime);
338 auto newSum = calcSha1sum(file);
339 if (newSum == sha1sum) {
346 if (adjustSha1(it, entry, newSum)) {
353 }
catch (FileException&) {
363File FilePoolCore::scanDirectory(
364 const Sha1Sum& sha1sum,
const std::string& directory, std::string_view poolPath,
365 ScanProgress& progress)
373 assert(!result.is_open());
376 result = scanFile(sha1sum, path, st, poolPath, progress);
377 return !result.is_open();
383File FilePoolCore::scanFile(
const Sha1Sum& sha1sum,
const std::string& filename,
385 ScanProgress& progress)
387 ++progress.amountScanned;
390 if (now > (progress.lastTime + 250'000)) {
391 progress.lastTime = now;
392 progress.printed =
true;
394 "Searching for file with sha1sum ", sha1sum.toString(),
395 "...\nIndexing filepool ", poolPath,
": [",
396 progress.amountScanned,
"]: ",
397 std::string_view(filename).substr(poolPath.size())),
402 if (
auto [idx, entry] = findInDatabase(filename); idx == Index(-1)) {
406 auto sum = calcSha1sum(file);
407 insert(
sum, time, filename);
408 if (
sum == sha1sum) {
411 }
catch (FileException&) {
416 assert(filename == entry->filename);
418 if (entry->getTime() == time) {
420 if (entry->sum == sha1sum) {
421 return File(filename);
426 auto sum = calcSha1sum(file);
427 entry->setTime(time);
428 adjustSha1(idx, *entry,
sum);
429 if (
sum == sha1sum) {
433 }
catch (FileException&) {
441std::pair<FilePoolCore::Index, FilePoolCore::Entry*> FilePoolCore::findInDatabase(std::string_view filename)
443 auto it = filenameIndex.
find(filename);
444 if (!it)
return {Index(-1),
nullptr};
447 auto& entry = pool[idx];
448 assert(entry.filename == filename);
452 return {Index(-1),
nullptr};
454 return {idx, &entry};
460 const std::string& filename = file.
getURL();
462 auto [idx, entry] = findInDatabase(filename);
463 if (idx != Index(-1)) {
464 if (entry->getTime() == time) {
472 auto sum = calcSha1sum(file);
473 if (idx == Index(-1)) {
475 insert(
sum, time, filename);
478 entry->setTime(time);
479 adjustSha1(idx, *entry,
sum);
EmplaceResult emplace(Args &&...args)
bool insert(Value2 &&val)
Value * find(const Value2 &val) const
bool erase(const Value2 &val)
Sha1Sum getSha1Sum(File &file)
Calculate sha1sum for the given File object.
File getFile(FileType fileType, const Sha1Sum &sha1sum)
Search file with the given sha1sum.
std::vector< Dir > Directories
FilePoolCore(std::string fileCache, std::function< Directories()> getDirectories, std::function< void(std::string_view, float)> reportProgress)
std::span< const uint8_t > mmap()
Map file in memory.
std::string_view getOriginalName()
Get Original filename for this object.
time_t getModificationDate()
Get the date/time of last modification.
bool is_open() const
Return true iff this file handle refers to an open file.
const std::string & getURL() const
Returns the URL of this file object.
void resize(size_t size)
Grow or shrink the memory block.
bool empty() const
No memory allocated?
const T * data() const
Returns pointer to the start of the memory buffer.
Helper class to perform a sha1 calculation.
Sha1Sum digest()
Get the final hash.
void update(std::span< const uint8_t > data)
Incrementally calculate the hash value.
This class represents the result of a sha1 calculation (a 160-bit value).
std::string toString() const
mat4 rotate(float angle, const vec3 &axis)
std::string toString(time_t time)
time_t fromString(std::span< const char, 24 > s)
constexpr time_t INVALID_TIME_T
string expandTilde(string path)
Expand the '~' character to the users home directory.
void openOfStream(std::ofstream &stream, zstring_view filename)
Open an ofstream in a platform-independent manner.
time_t getModificationDate(const Stat &st)
Get the date/time of last modification.
uint64_t getTime()
Get current (real) time in us.
This file implemented 3 utility functions:
bool foreach_file_recursive(std::string path, FileAction fileAction)
bool is_sorted(ForwardRange &&range, Compare comp={}, Proj proj={})
auto find_if(InputRange &&range, UnaryPredicate pred)
auto upper_bound(ForwardRange &&range, const T &value, Compare comp={}, Proj proj={})
auto equal_range(ForwardRange &&range, const T &value, Compare comp={})
constexpr void sort(RandomAccessRange &&range)
size_t size(std::string_view utf8)
auto distance(octet_iterator first, octet_iterator last)
auto sum(InputRange &&range, Proj proj={})
TemporaryString tmpStrCat(Ts &&... ts)
const Sha1Sum & operator()(FilePoolCore::Index idx) const
const FilePoolCore::Pool & pool
constexpr auto begin(const zstring_view &x)