29 std::function<
void(std::string_view,
float)> reportProgress_)
30 : fileCache(
std::move(fileCache_))
31 , getDirectories(
std::move(getDirectories_))
32 , reportProgress(
std::move(reportProgress_))
48void FilePoolCore::insert(
const Sha1Sum&
sum, time_t time,
const std::string& filename)
50 stringBuffer.push_back(filename);
51 auto idx = pool.
emplace(
sum, time, stringBuffer.back()).idx;
53 sha1Index.insert(it, idx);
58FilePoolCore::Sha1Index::iterator FilePoolCore::getSha1Iterator(Index idx,
const Entry& entry)
66 assert(
false);
return sha1Index.end();
69void FilePoolCore::remove(Sha1Index::iterator it)
72 filenameIndex.
erase(idx);
78void FilePoolCore::remove(Index idx,
const Entry& entry)
80 remove(getSha1Iterator(idx, entry));
83void FilePoolCore::remove(Index idx)
85 remove(idx, pool[idx]);
94bool FilePoolCore::adjustSha1(Sha1Index::iterator it, Entry& entry,
const Sha1Sum& newSum)
101 rotate(it, it + 1, newIt);
106 rotate(newIt, it, it + 1);
116bool FilePoolCore::adjustSha1(Index idx, Entry& entry,
const Sha1Sum& newSum)
118 return adjustSha1(getSha1Iterator(idx, entry), entry, newSum);
121time_t FilePoolCore::Entry::getTime()
129void FilePoolCore::Entry::setTime(time_t
t)
136static std::optional<std::tuple<Sha1Sum, const char*, std::string_view>> parse(
137 std::span<char> line)
139 if (line.size() <= 68)
return {};
142 if (line[40] !=
' ')
return {};
143 if (line[41] !=
' ')
return {};
144 if (line[45] !=
' ')
return {};
145 if (line[49] !=
' ')
return {};
146 if (line[52] !=
' ')
return {};
147 if (line[55] !=
':')
return {};
148 if (line[58] !=
':')
return {};
149 if (line[61] !=
' ')
return {};
150 if (line[66] !=
' ')
return {};
151 if (line[67] !=
' ')
return {};
153 Sha1Sum sha1(Sha1Sum::UninitializedTag{});
155 sha1.parse40(subspan<40>(line));
156 }
catch (MSXException&) {
160 const char* timeStr = &line[42];
163 std::string_view filename(&line[68], line.size() - 68);
165 return std::tuple{sha1, timeStr, filename};
168void FilePoolCore::readSha1sums()
170 assert(sha1Index.empty());
171 assert(fileMem.
empty());
173 File file(fileCache);
174 auto size = file.getSize();
176 file.read(std::span{fileMem.
data(), size});
177 fileMem[
size] =
'\n';
181 char* data = fileMem.
data();
182 char* data_end = data +
size + 1;
183 while (data != data_end) {
185 auto* it =
static_cast<char*
>(memchr(data,
'\n', data_end - data));
186 if (it ==
nullptr) it = data_end;
187 if ((it != data) && (it[-1] ==
'\r')) --it;
189 if (
auto r = parse({data, it})) {
190 auto [
sum, timeStr, filename] = *r;
191 sha1Index.push_back(pool.
emplace(
sum, timeStr, filename).idx);
195 data = std::find_if(it + 1, data_end, [](
char c) {
196 return c !=
one_of(
'\n',
'\r');
209 auto n = sha1Index.size();
213 Index idx = sha1Index[n];
214 bool inserted = filenameIndex.
insert(idx);
223void FilePoolCore::writeSha1sums()
227 if (!file.is_open()) {
230 for (
auto idx : sha1Index) {
231 const auto& entry = pool[idx];
232 file << entry.sum.toString() <<
" ";
234 file << entry.timeStr;
239 file <<
" " << entry.filename <<
'\n';
245 File result = getFromPool(sha1sum);
246 if (result.
is_open())
return result;
250 ScanProgress progress {
254 for (
const auto& [path, types] : getDirectories()) {
258 if (progress.printed) {
266 if (progress.printed) {
267 reportProgress(
tmpStrCat(
"Did not find file with sha1sum ", sha1sum.
toString()), 1.0f);
276 constexpr size_t STEP_SIZE = 1024 * 1024;
278 auto data = file.
mmap();
281 size_t size = data.size();
283 size_t remaining = size;
285 bool everShowedProgress =
false;
287 auto report = [&](
float fraction) {
292 while (remaining > STEP_SIZE) {
293 sha1.
update({&data[done], STEP_SIZE});
295 remaining -= STEP_SIZE;
298 if ((now - lastShowedProgress) > 250'000) {
299 report(
float(done) /
float(size));
300 lastShowedProgress = now;
301 everShowedProgress =
true;
306 sha1.
update({&data[done], remaining});
308 if (everShowedProgress) {
314File FilePoolCore::getFromPool(
const Sha1Sum& sha1sum)
321 auto it =
begin(sha1Index) + i;
322 auto& entry = pool[*it];
331 File file(std::string(entry.filename));
333 if (entry.getTime() == newTime) {
339 entry.setTime(newTime);
341 auto newSum = calcSha1sum(file);
342 if (newSum == sha1sum) {
349 if (adjustSha1(it, entry, newSum)) {
356 }
catch (FileException&) {
366File FilePoolCore::scanDirectory(
367 const Sha1Sum& sha1sum,
const std::string& directory, std::string_view poolPath,
368 ScanProgress& progress)
376 assert(!result.is_open());
379 result = scanFile(sha1sum, path, st, poolPath, progress);
380 return !result.is_open();
386File FilePoolCore::scanFile(
const Sha1Sum& sha1sum,
const std::string& filename,
388 ScanProgress& progress)
390 ++progress.amountScanned;
393 now > (progress.lastTime + 250'000)) {
394 progress.lastTime = now;
395 progress.printed =
true;
397 "Searching for file with sha1sum ", sha1sum.toString(),
398 "...\nIndexing filepool ", poolPath,
": [",
399 progress.amountScanned,
"]: ",
400 std::string_view(filename).substr(poolPath.size())),
405 if (
auto [idx, entry] = findInDatabase(filename); idx == Index(-1)) {
409 auto sum = calcSha1sum(file);
410 insert(
sum, time, filename);
411 if (
sum == sha1sum) {
414 }
catch (FileException&) {
419 assert(filename == entry->filename);
421 if (entry->getTime() == time) {
423 if (entry->sum == sha1sum) {
424 return File(filename);
429 auto sum = calcSha1sum(file);
430 entry->setTime(time);
431 adjustSha1(idx, *entry,
sum);
432 if (
sum == sha1sum) {
436 }
catch (FileException&) {
444std::pair<FilePoolCore::Index, FilePoolCore::Entry*> FilePoolCore::findInDatabase(std::string_view filename)
446 auto it = filenameIndex.
find(filename);
447 if (!it)
return {Index(-1),
nullptr};
450 auto& entry = pool[idx];
451 assert(entry.filename == filename);
455 return {Index(-1),
nullptr};
457 return {idx, &entry};
463 const std::string& filename = file.
getURL();
465 auto [idx, entry] = findInDatabase(filename);
466 if ((idx != Index(-1)) && (entry->getTime() == time)) {
473 auto sum = calcSha1sum(file);
474 if (idx == Index(-1)) {
476 insert(
sum, time, filename);
479 entry->setTime(time);
480 adjustSha1(idx, *entry,
sum);
EmplaceResult emplace(Args &&...args)
Value * find(const Value2 &val) const
bool insert(Value2 &&val)
bool erase(const Value2 &val)
Sha1Sum getSha1Sum(File &file)
Calculate sha1sum for the given File object.
File getFile(FileType fileType, const Sha1Sum &sha1sum)
Search file with the given sha1sum.
std::vector< Dir > Directories
FilePoolCore(std::string fileCache, std::function< Directories()> getDirectories, std::function< void(std::string_view, float)> reportProgress)
std::span< const uint8_t > mmap()
Map file in memory.
std::string_view getOriginalName()
Get Original filename for this object.
time_t getModificationDate()
Get the date/time of last modification.
bool is_open() const
Return true iff this file handle refers to an open file.
const std::string & getURL() const
Returns the URL of this file object.
void resize(size_t size)
Grow or shrink the memory block.
bool empty() const
No memory allocated?
const T * data() const
Returns pointer to the start of the memory buffer.
Helper class to perform a sha1 calculation.
Sha1Sum digest()
Get the final hash.
void update(std::span< const uint8_t > data)
Incrementally calculate the hash value.
This class represents the result of a sha1 calculation (a 160-bit value).
std::string toString() const
mat4 rotate(float angle, const vec3 &axis)
std::string toString(time_t time)
time_t fromString(std::span< const char, 24 > s)
constexpr time_t INVALID_TIME_T
string expandTilde(string path)
Expand the '~' character to the users home directory.
void openOfStream(std::ofstream &stream, zstring_view filename)
Open an ofstream in a platform-independent manner.
time_t getModificationDate(const Stat &st)
Get the date/time of last modification.
uint64_t getTime()
Get current (real) time in us.
This file implemented 3 utility functions:
bool foreach_file_recursive(std::string path, FileAction fileAction)
bool is_sorted(ForwardRange &&range, Compare comp={}, Proj proj={})
auto upper_bound(ForwardRange &&range, const T &value, Compare comp={}, Proj proj={})
auto equal_range(ForwardRange &&range, const T &value, Compare comp={})
constexpr void sort(RandomAccessRange &&range)
size_t size(std::string_view utf8)
auto distance(octet_iterator first, octet_iterator last)
constexpr auto sum(InputRange &&range, Proj proj={})
TemporaryString tmpStrCat(Ts &&... ts)
const Sha1Sum & operator()(FilePoolCore::Index idx) const
const FilePoolCore::Pool & pool
constexpr auto begin(const zstring_view &x)